{
  "schema_version": 2,
  "adapter_key": "skilldreamer",
  "manifest_ref": "reproduction:skilldreamer",
  "seeds": [
    42
  ],
  "result": "negative_validation_csi_protocol_diagnostic",
  "measured_locally": true,
  "diagnostic_only": true,
  "promotion_eligible": false,
  "not_gpu_receipt": true,
  "gpu_validation_passed": false,
  "capability_receipt": false,
  "accelerator": {
    "vendor": "NVIDIA",
    "model": "NVIDIA A100-SXM4-80GB"
  },
  "metrics": {
    "wall_seconds": 596.9424207285047,
    "model_calls": 210,
    "generated_tokens": 29194,
    "validation_episodes": 8,
    "psg_episodes": 7,
    "hsr_episodes": 7,
    "actual_csi_assessments": 2,
    "loaded_skill_episodes": 2,
    "executed_actions": 0,
    "runtime_csi_calls": 168,
    "submitted_answers": 0
  },
  "execution_checks": {
    "psg_episodes": 7,
    "hsr_episodes": 7,
    "actual_csi_assessments": 2,
    "loaded_skill_episodes": 2,
    "executed_actions": 0
  },
  "dataset": {
    "kind": "public-resource",
    "public_id": "openai/grade-school-math",
    "revision": "3101c7d5072418e28b9008a6636bde82a006892c",
    "source_split": "train",
    "scope": "fixed_original_validation_only",
    "validation_ids": [
      "13",
      "1",
      "4",
      "6",
      "23",
      "21",
      "14",
      "15"
    ],
    "parent_24_rows_sha256": "8f33a9f7a44067e362458e4daf0eb1ad629a2aac2f44ffefe731e36b19dcb8c1",
    "transformation": "select original fixed validation IDs; keep only id and question; discard every answer before probing",
    "gold_present": false,
    "scorer_present": false,
    "evaluation_ids_used": false,
    "official_test_opened": false,
    "files": {
      "questions.jsonl": "6a3213a1941882e4d0787b903df170452f13ea13f52ad9dbc4e6cb6e418eca5e"
    }
  },
  "checkpoint": {
    "public_id": "Qwen/Qwen2.5-7B-Instruct",
    "revision": "bb46c15ee4bb56c5b63245ef50fd7637234d6f75",
    "files": {
      "config.json": "7463bb0ea78315365e6c6b74de4e73bbcc8359dfb0c5a737584e077d42c0b03c",
      "generation_config.json": "3a8f9087e486054c8a4a08dae2e5a3ba62e23da212b5b8c08bc42cb983c3459f",
      "merges.txt": "599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3",
      "model.safetensors.index.json": "624bf7c47cd12468fdc16e38a47cf4f19e0415b859a223ba3c027eed2f0e1028",
      "tokenizer.json": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539",
      "tokenizer_config.json": "5b5d4f65d0acd3b2d56a35b56d374a36cbc1c8fa5cf3b3febbbfabf22f359583",
      "vocab.json": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910",
      "model-00001-of-00004.safetensors": "a1333e6293854747c481288ea83b348226af178dd565c49b6f9495ba1966aba7",
      "model-00002-of-00004.safetensors": "f5d25a2772cb825164a2a2c0fb6d51a87e282abf21e4dd75bc5cfb3cd0ea6185",
      "model-00003-of-00004.safetensors": "8efdec4c1bc12317ae1a38dc42b595ce777738a64deea3fcb8a0a91381bcdfd5",
      "model-00004-of-00004.safetensors": "1a72d403cdf0c1ec3cb7f289f17b394a01e64394c2e9b3c0f94dbce3faf879bd"
    }
  },
  "encoder": {
    "public_id": "sentence-transformers/all-MiniLM-L6-v2",
    "revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
    "files": {
      "model.safetensors": "53aa51172d142c89d9012cce15ae4d6cc0ca6895895114379cacb4fab128d9db",
      "config.json": "953f9c0d463486b10a6871cc2fd59f223b2c70184f49815e7efbcab5d8908b41",
      "tokenizer.json": "be50c3628f2bf5bb5e3a7f17b1f74611b2561a3a27eeab05e5aa30f411572037",
      "tokenizer_config.json": "acb92769e8195aabd29b7b2137a9e6d6e25c476a4f15aa4355c233426c61576b",
      "special_tokens_map.json": "303df45a03609e4ead04bc3dc1536d0ab19b5358db685b6f3da123d05ec200e3",
      "vocab.txt": "07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3"
    }
  },
  "budget": {
    "seed": 42,
    "max_new_tokens": 256,
    "action_budget": 8,
    "top_k": 3,
    "question_limit": 8,
    "calls_limit_per_episode": 30,
    "calls_limit_total": 240,
    "total_wall_limit_seconds": 600
  },
  "raw_artifacts": {
    "seed42.json": {
      "artifact_path": "docs/reproductions/2609.01642-skilldreamer/metrics/raw/validation-negative-d8603f65/seed42.json",
      "sha256": "41bed039696937c502509e8a3bd8b5ec41f8a5acb78554c9c1ffe10c2e1c92a3"
    },
    "seed42.partial.json": {
      "artifact_path": "docs/reproductions/2609.01642-skilldreamer/metrics/raw/validation-negative-d8603f65/seed42.partial.json",
      "sha256": "ed9a60f4b8862fb944e3a41288f57a9ee47020847664a3fae25bbef593e1c3e9"
    },
    "seed42.trace.jsonl": {
      "artifact_path": "docs/reproductions/2609.01642-skilldreamer/metrics/raw/validation-negative-d8603f65/seed42.trace.jsonl",
      "sha256": "07296f521f5d8751237268b96596485217a4e34f81736f4067ebcb2588946259"
    },
    "trace-analysis.json": {
      "artifact_path": "docs/reproductions/2609.01642-skilldreamer/metrics/raw/validation-negative-d8603f65/trace-analysis.json",
      "sha256": "903f02afa981a557d6a385894946a96013c90a51c0a60cf312b84ffa06faae30"
    }
  },
  "evaluation_protocol": {
    "tier": "l1_core_mechanism",
    "seeds": [
      42
    ],
    "formal_comparison": false,
    "validation_only": true,
    "validation_ids": [
      "13",
      "1",
      "4",
      "6",
      "23",
      "21",
      "14",
      "15"
    ],
    "gold_present": false,
    "scorer_present": false,
    "evaluation_ids_used": false,
    "official_test_opened": false,
    "test_used_for_selection": false,
    "claim_policy": "negative question-only validation formatting diagnostic; not capability evidence or passed GPU acceptance"
  },
  "diagnosis": {
    "runtime_categories": {
      "missing_or_invalid_action_json": 12,
      "csi_label_order_or_missing_line": 79,
      "csi_invalid_decision_enum": 65,
      "csi_empty_field": 10,
      "valid_csi_gallery_load_then_requery": 2
    },
    "episodes": [
      {
        "id": "13",
        "model_calls": 1,
        "actual_generated_tokens": 208,
        "runtime_calls": 0,
        "runtime_categories": {},
        "visible_cap_calls_all_stages": [],
        "post_load_categories": {},
        "max_input_tokens": 136,
        "executed_actions": 0,
        "failure_stage": "psg_json",
        "cpu_exact_trace_replay": false
      },
      {
        "id": "1",
        "model_calls": 30,
        "actual_generated_tokens": 4309,
        "runtime_calls": 24,
        "runtime_categories": {
          "missing_or_invalid_action_json": 5,
          "csi_label_order_or_missing_line": 18,
          "csi_invalid_decision_enum": 1
        },
        "visible_cap_calls_all_stages": [
          3,
          4,
          5,
          6,
          7,
          8
        ],
        "post_load_categories": {},
        "max_input_tokens": 782,
        "executed_actions": 0,
        "failure_stage": "runtime_csi_schema_call_limit",
        "cpu_exact_trace_replay": true
      },
      {
        "id": "4",
        "model_calls": 30,
        "actual_generated_tokens": 4153,
        "runtime_calls": 24,
        "runtime_categories": {
          "csi_label_order_or_missing_line": 13,
          "csi_invalid_decision_enum": 8,
          "csi_empty_field": 2,
          "valid_csi_gallery_load_then_requery": 1
        },
        "visible_cap_calls_all_stages": [
          33,
          34,
          35,
          36,
          37,
          39
        ],
        "post_load_categories": {
          "csi_label_order_or_missing_line": 5,
          "csi_invalid_decision_enum": 5
        },
        "max_input_tokens": 809,
        "executed_actions": 0,
        "failure_stage": "runtime_csi_schema_call_limit",
        "cpu_exact_trace_replay": true
      },
      {
        "id": "6",
        "model_calls": 30,
        "actual_generated_tokens": 3353,
        "runtime_calls": 24,
        "runtime_categories": {
          "csi_label_order_or_missing_line": 9,
          "missing_or_invalid_action_json": 7,
          "csi_invalid_decision_enum": 7,
          "valid_csi_gallery_load_then_requery": 1
        },
        "visible_cap_calls_all_stages": [
          63,
          64,
          65,
          66
        ],
        "post_load_categories": {
          "csi_invalid_decision_enum": 1
        },
        "max_input_tokens": 822,
        "executed_actions": 0,
        "failure_stage": "runtime_csi_schema_call_limit",
        "cpu_exact_trace_replay": true
      },
      {
        "id": "23",
        "model_calls": 30,
        "actual_generated_tokens": 3920,
        "runtime_calls": 24,
        "runtime_categories": {
          "csi_invalid_decision_enum": 14,
          "csi_empty_field": 6,
          "csi_label_order_or_missing_line": 4
        },
        "visible_cap_calls_all_stages": [
          93,
          94,
          95,
          96,
          97
        ],
        "post_load_categories": {},
        "max_input_tokens": 760,
        "executed_actions": 0,
        "failure_stage": "runtime_csi_schema_call_limit",
        "cpu_exact_trace_replay": true
      },
      {
        "id": "21",
        "model_calls": 30,
        "actual_generated_tokens": 3939,
        "runtime_calls": 24,
        "runtime_categories": {
          "csi_empty_field": 2,
          "csi_label_order_or_missing_line": 4,
          "csi_invalid_decision_enum": 18
        },
        "visible_cap_calls_all_stages": [
          123,
          124,
          125,
          126,
          127
        ],
        "post_load_categories": {},
        "max_input_tokens": 743,
        "executed_actions": 0,
        "failure_stage": "runtime_csi_schema_call_limit",
        "cpu_exact_trace_replay": true
      },
      {
        "id": "14",
        "model_calls": 30,
        "actual_generated_tokens": 5194,
        "runtime_calls": 24,
        "runtime_categories": {
          "csi_label_order_or_missing_line": 12,
          "csi_invalid_decision_enum": 12
        },
        "visible_cap_calls_all_stages": [
          153,
          154,
          155,
          156,
          157,
          158,
          163,
          167,
          168,
          178,
          180
        ],
        "post_load_categories": {},
        "max_input_tokens": 762,
        "executed_actions": 0,
        "failure_stage": "runtime_csi_schema_call_limit",
        "cpu_exact_trace_replay": true
      },
      {
        "id": "15",
        "model_calls": 29,
        "actual_generated_tokens": 4118,
        "runtime_calls": 24,
        "runtime_categories": {
          "csi_label_order_or_missing_line": 19,
          "csi_invalid_decision_enum": 5
        },
        "visible_cap_calls_all_stages": [
          183,
          184,
          185,
          186
        ],
        "post_load_categories": {},
        "max_input_tokens": 771,
        "executed_actions": 0,
        "failure_stage": "runtime_csi_schema_call_limit",
        "cpu_exact_trace_replay": true
      }
    ],
    "visible_cap_counts": {
      "pseudo_skill": 33,
      "csi": 8
    },
    "max_input_tokens_before_truncation": 822,
    "complete_valid_csi_with_rejected_action_json": 0,
    "interface_gap": "Original prompt omitted explicit nonempty LABEL: value formatting and complete DECISION first-word enum; no parser bug established.",
    "post_load": "ID4 loaded rates at call51 then 10 schema failures; ID6 loaded arithmetic at call90 then one enum failure. All post-load responses below 256 visible tokens; no normal finish.",
    "cpu_fixture_scope": "Exact seven-episode trace replay and compliant calculator/finish fixture are diagnostics only, not model capability evidence."
  },
  "provenance": {
    "code_commit": "0f5dca9f7f0319c6e3bb57aa578cfe26499a6e7e",
    "probe_source": {
      "commit": "d8603f65b207e5345f156621f840c69a24ecace2",
      "sha256": "7799029a67ebd7918bb69db94c86d6a7c1b94558f9510e4eb9581b9c9c1d3d9f",
      "immutable_root_bound": true
    },
    "artifact_path": "docs/reproductions/2609.01642-skilldreamer/metrics/public-a100-validation-negative-d8603f65.json",
    "executed_source_sha256": {
      "skilldreamer/algorithm.py": "d6e2e446a39826d3b3c4c29691e342599bad5272a9330cb20c666ebbb8500345",
      "skilldreamer/experiment.py": "5d327072480d34b940d3aebbb4f6a638dffb81fb6c342c09a799ad8eeb8524c8",
      "near/data.py": "795416bdffeb25880aeaf85b4bfd6480b8677b45cddabe37ba2e49d7bf544eb8",
      "near/experiment.py": "197e3fefad06f1f0e54533b608c5d717e57d0b6a919a105f88eee79447d94399",
      "state_opd/environment.py": "0e9564f9ec540985eb010b1e70f46938acc1bd946041b518d6579253762080e1",
      "skilldreamer/gallery.json": "9ff5e28de7d2eb7b3581ef01bbee66b0d201a543033bde6a2aa66da37f687970"
    },
    "algorithm_archive_sha256": "3d9ce51e7d42813e3b3aa6f204736ea7755b8e29e417b248667db9a8810502d6",
    "probe_archive_sha256": "a1eede5ea2afcaa6919125d4ef4db674b3116c4949aeb59d1c91a5989d0b1d81",
    "dataset_fingerprint": "8b54e0b9206e2ee831ca074d0661ae75cf5d2212ebefc42f6c613591eed5db62",
    "raw_result_sha256": "41bed039696937c502509e8a3bd8b5ec41f8a5acb78554c9c1ffe10c2e1c92a3"
  },
  "limitations": [
    "Probe completion is not core-chain completion: zero actual tool actions; no passed receipt.",
    "Eight fixed train-derived validation questions, one seed, no gold/scorer/test; no formal capability comparison.",
    "Historical 0.5B and this original-prompt 7B negative batches retained; no relabeling, parser relaxation or budget changes.",
    "Prompt-interface clarification is a separate unexecuted source change, not a measured improvement."
  ]
}