{
  "schema_version": 2,
  "manifest_ref": "multimodal-models:scienceqa",
  "benchmark": "scienceqa",
  "evaluation_tier": "l2_public_benchmark",
  "seeds": [
    42
  ],
  "evaluated_examples": 500,
  "prediction_source": "public_checkpoint",
  "metadata": {
    "dataset_source": "https://github.com/lupantech/ScienceQA",
    "annotations_sha256": "992b2e3c5296a59d36135471c66875069ca3747e0245af79eeb5752f976d3f36",
    "split": "test",
    "subset_protocol": "first 500 ids in the official test split",
    "maximum_examples": 500,
    "model_id": "HuggingFaceTB/SmolVLM2-256M-Video-Instruct",
    "model_revision": "067788b187b95ebe7b2e040b3e4299e342e5b8fd",
    "checkpoint_committed": false,
    "decoding": {
      "do_sample": false,
      "max_new_tokens": 16,
      "prompt": "context + question + lettered choices; answer only the option letter"
    },
    "runtime": {
      "platform": "Linux x86_64",
      "resolved_device": "cuda",
      "accelerator": "NVIDIA A30",
      "cuda_version": "12.8"
    },
    "mac_smoke": {
      "platform": "macOS arm64",
      "device": "cpu",
      "examples": 1,
      "checkpoint_loaded": true
    }
  },
  "seed_results": [
    {
      "seed": 42,
      "accuracy": 0.568,
      "image_accuracy": 0.6286919831223629,
      "text_accuracy": 0.5133079847908745,
      "coverage": 1.0,
      "parse_rate": 0.998
    }
  ],
  "aggregate_metrics": {
    "accuracy": {
      "mean": 0.568,
      "std": 0.0,
      "ci95": null,
      "n": 1
    },
    "image_accuracy": {
      "mean": 0.6286919831223629,
      "std": 0.0,
      "ci95": null,
      "n": 1
    },
    "text_accuracy": {
      "mean": 0.5133079847908745,
      "std": 0.0,
      "ci95": null,
      "n": 1
    },
    "coverage": {
      "mean": 1.0,
      "std": 0.0,
      "ci95": null,
      "n": 1
    },
    "parse_rate": {
      "mean": 0.998,
      "std": 0.0,
      "ci95": null,
      "n": 1
    }
  },
  "formal_comparison": false,
  "claim_policy": "single deterministic zero-shot checkpoint run on a fixed 500-example subset; not a full-test leaderboard or multi-seed improvement claim",
  "evaluation_protocol": {
    "tier": "l2_public_benchmark",
    "seeds": [
      42
    ],
    "formal_comparison": false,
    "claim_policy": "single deterministic zero-shot checkpoint run on a fixed 500-example subset; not a full-test leaderboard or multi-seed improvement claim"
  },
  "provenance": {
    "created_at": "2026-08-11T06:40:17.816253+00:00",
    "artifact_path": "docs/multimodal-models/metrics/scienceqa-smolvlm2-256m-500.json",
    "dataset_fingerprint": "sha256:992b2e3c5296a59d36135471c66875069ca3747e0245af79eeb5752f976d3f36"
  }
}