{
  "schema_version": 2,
  "method": "alps",
  "manifest_ref": "foundation-models:alps",
  "dataset": {
    "id": "synthetic:oct22-bulletin-alps-v1",
    "revision": "sha256:1dc0f401f242d6bea9435763cc2023d4f84d134164e3386e3f6ad8052696c716",
    "recipe": "source-frozen-generated-mechanism-contract-inputs",
    "capability_benchmark": false
  },
  "seeds": [
    42,
    43,
    44
  ],
  "per_seed": [
    {
      "paper": "2610.30120v1",
      "seed": 42,
      "device": "cuda",
      "evaluation_tier": "l1_mechanism",
      "diagnostic_only": true,
      "promotion_eligible": false,
      "evolve_integrated": false,
      "full_experiment_status": "not_reproduced_high_cost",
      "pretrained_model_calls": 0,
      "dataset": "generated-crop-query-token-contract-suite",
      "upstream_revision": "30ab44b9fcdf3253fc67f5a53e31ea5141eb45c1",
      "recipe": {
        "batch": 2,
        "grid": 4,
        "query_width": 8,
        "token_channels": 6,
        "predictor": "test-only linear readout, not the ALPS transformer"
      },
      "checks": {
        "independent_formula_match": true,
        "finite_student_query_gradients": true,
        "frozen_teacher_has_no_gradient": true,
        "masked_patch_gradient_is_zero": true,
        "remove_overlap_mask_changes_loss": true,
        "remove_fixed_statistics_changes_loss": true,
        "remove_geometry_changes_queries": true
      },
      "metrics": {
        "formula_absolute_error": 3.814697265625e-06,
        "overlap_patch_count": 13.0,
        "removed_overlap_loss_delta": 1.5497360229492188,
        "removed_standardization_loss_delta": 3.263427734375,
        "removed_geometry_query_mse": 1.154480218887329,
        "masked_patch_gradient_max": 0.0,
        "query_parameter_gradient_norm": 20.680503845214844,
        "pretrained_model_calls": 0.0
      }
    },
    {
      "paper": "2610.30120v1",
      "seed": 43,
      "device": "cuda",
      "evaluation_tier": "l1_mechanism",
      "diagnostic_only": true,
      "promotion_eligible": false,
      "evolve_integrated": false,
      "full_experiment_status": "not_reproduced_high_cost",
      "pretrained_model_calls": 0,
      "dataset": "generated-crop-query-token-contract-suite",
      "upstream_revision": "30ab44b9fcdf3253fc67f5a53e31ea5141eb45c1",
      "recipe": {
        "batch": 2,
        "grid": 4,
        "query_width": 8,
        "token_channels": 6,
        "predictor": "test-only linear readout, not the ALPS transformer"
      },
      "checks": {
        "independent_formula_match": true,
        "finite_student_query_gradients": true,
        "frozen_teacher_has_no_gradient": true,
        "masked_patch_gradient_is_zero": true,
        "remove_overlap_mask_changes_loss": true,
        "remove_fixed_statistics_changes_loss": true,
        "remove_geometry_changes_queries": true
      },
      "metrics": {
        "formula_absolute_error": 0.0,
        "overlap_patch_count": 13.0,
        "removed_overlap_loss_delta": 1.4435348510742188,
        "removed_standardization_loss_delta": 5.389125823974609,
        "removed_geometry_query_mse": 1.1560008525848389,
        "masked_patch_gradient_max": 0.0,
        "query_parameter_gradient_norm": 13.932915687561035,
        "pretrained_model_calls": 0.0
      }
    },
    {
      "paper": "2610.30120v1",
      "seed": 44,
      "device": "cuda",
      "evaluation_tier": "l1_mechanism",
      "diagnostic_only": true,
      "promotion_eligible": false,
      "evolve_integrated": false,
      "full_experiment_status": "not_reproduced_high_cost",
      "pretrained_model_calls": 0,
      "dataset": "generated-crop-query-token-contract-suite",
      "upstream_revision": "30ab44b9fcdf3253fc67f5a53e31ea5141eb45c1",
      "recipe": {
        "batch": 2,
        "grid": 4,
        "query_width": 8,
        "token_channels": 6,
        "predictor": "test-only linear readout, not the ALPS transformer"
      },
      "checks": {
        "independent_formula_match": true,
        "finite_student_query_gradients": true,
        "frozen_teacher_has_no_gradient": true,
        "masked_patch_gradient_is_zero": true,
        "remove_overlap_mask_changes_loss": true,
        "remove_fixed_statistics_changes_loss": true,
        "remove_geometry_changes_queries": true
      },
      "metrics": {
        "formula_absolute_error": 0.0,
        "overlap_patch_count": 13.0,
        "removed_overlap_loss_delta": 2.7219924926757812,
        "removed_standardization_loss_delta": 5.088640213012695,
        "removed_geometry_query_mse": 1.8741462230682373,
        "masked_patch_gradient_max": 0.0,
        "query_parameter_gradient_norm": 7.87799596786499,
        "pretrained_model_calls": 0.0
      }
    }
  ],
  "metrics": {
    "formula_absolute_error": {
      "mean": 1.2715657552083333e-06,
      "std": 2.202416493185523e-06,
      "n": 3
    },
    "masked_patch_gradient_max": {
      "mean": 0.0,
      "std": 0.0,
      "n": 3
    },
    "overlap_patch_count": {
      "mean": 13.0,
      "std": 0.0,
      "n": 3
    },
    "pretrained_model_calls": {
      "mean": 0.0,
      "std": 0.0,
      "n": 3
    },
    "query_parameter_gradient_norm": {
      "mean": 14.16380516688029,
      "std": 6.404376195317269,
      "n": 3
    },
    "removed_geometry_query_mse": {
      "mean": 1.3948757648468018,
      "std": 0.4150610884852362,
      "n": 3
    },
    "removed_overlap_loss_delta": {
      "mean": 1.9050877888997395,
      "std": 0.7094502403378778,
      "n": 3
    },
    "removed_standardization_loss_delta": {
      "mean": 4.580397923787435,
      "std": 1.1503828755894958,
      "n": 3
    }
  },
  "evaluation_tier": "l1_mechanism",
  "evaluation_protocol": {
    "tier": "l1_mechanism",
    "seeds": [
      42,
      43,
      44
    ],
    "formal_comparison": false,
    "claim_policy": "generated mechanism/scorer contracts, no pretrained capability or full-paper comparison"
  },
  "provenance": {
    "commit": "d767be53ac5a8a79f2b02aa23db0fcf3773ae588",
    "artifact_path": "docs/foundation-models/2610.30120-alps/metrics/component-cuda-seeds42-44.json",
    "accelerator": "NVIDIA A100-SXM4-80GB",
    "executed_source_sha256": {
      "scripts/run_oct22_bulletin_components.py": "0581649a3e872529905359709a0a7d21b44bbb5ce129b43cd4f65df75895b323",
      "scripts/oct18_frozen_sources.py": "da08355049c1f6b5184ee659643c9be9decedf2041ef6a87a3dce2e4582e385a",
      "src/auto_research/foundation_models/alps.py": "92f71f68fe01d5489327c5d7d89afca43b453722ae4b785336f43d3934830269",
      "src/auto_research/agent_research/asteroid.py": "95fd728c6a80779d73f89e2d2ca9e96ad2ce2d6c94ac6994727df222452e4a5f",
      "src/auto_research/post_training/cord.py": "2616a3a9ea93bdbd1402b12be6112f3eef7cd08dc9a61c672156ed03470e5984",
      "src/auto_research/post_training/csd.py": "a4b08dc51e57cceec3a488cc0439de22b3212f8f3d0db21d79a381c047986739",
      "src/auto_research/system_one/multicorrect.py": "a6169b61c950ef7b5b40825afefefc3e7a02e018da4e4fc3b6d2c23d880a1406"
    },
    "raw_runner_artifact_sha256": "201500f7b1656614d340d30afc9892a1ebeb369e2865769c23cb7170457e1831",
    "dataset_fingerprint": "sha256:3d265a89be71e329c6996e86f7a3fb41402f6e7c2afc9068ac95b741a5fe4c3e"
  },
  "diagnostic_only": true,
  "fixture_demo": true,
  "pretrained_model_calls": 0,
  "formal_comparison": false,
  "promotion_eligible": false,
  "evolve_integrated": false,
  "full_experiment_status": "not_reproduced_high_cost",
  "full_experiment_review_done": true
}