{
  "schema_version": 2,
  "manifest_ref": "multimodal-models:cifar10-qa",
  "benchmark": "cifar10-qa",
  "evaluation_tier": "l1_public_images",
  "seeds": [
    42,
    43,
    44
  ],
  "evaluated_examples": 2000,
  "prediction_source": "micro_vlm_query; objective=cross_entropy; steps=300; train_examples=5000",
  "metadata": {
    "dataset_source": "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz",
    "dataset_checksum_md5": "c58f30108f718f92721af3b95e74349a",
    "dataset_license": "CIFAR-10 research dataset; see dataset website",
    "train_examples": 5000,
    "validation_examples": 1000,
    "test_examples": 1000,
    "architecture": "micro_vlm_query",
    "objective": "cross_entropy",
    "steps": 300,
    "dimensions": 192,
    "batch_size": 32,
    "learning_rate": 0.0003,
    "device": "cpu",
    "cpu_threads": 8,
    "parameters": 151114,
    "architecture_stats": {
      "connector": "query",
      "patch_size": 8,
      "visual_tokens": 16,
      "trainable_queries": 1
    }
  },
  "seed_results": [
    {
      "seed": 42,
      "validation_accuracy": 0.2029999941587448,
      "validation_shuffled_image_accuracy": 0.10400000214576721,
      "validation_blank_image_accuracy": 0.10000000149011612,
      "validation_visual_dependency_delta": 0.0989999920129776,
      "validation_blank_image_delta": 0.10299999266862869,
      "test_accuracy": 0.19200000166893005,
      "test_shuffled_image_accuracy": 0.09600000083446503,
      "test_blank_image_accuracy": 0.10000000149011612,
      "test_visual_dependency_delta": 0.09600000083446503,
      "test_blank_image_delta": 0.09200000017881393,
      "parameters": 151114
    },
    {
      "seed": 43,
      "validation_accuracy": 0.1860000044107437,
      "validation_shuffled_image_accuracy": 0.10199999809265137,
      "validation_blank_image_accuracy": 0.10000000149011612,
      "validation_visual_dependency_delta": 0.08400000631809235,
      "validation_blank_image_delta": 0.0860000029206276,
      "test_accuracy": 0.19699999690055847,
      "test_shuffled_image_accuracy": 0.10700000077486038,
      "test_blank_image_accuracy": 0.10000000149011612,
      "test_visual_dependency_delta": 0.08999999612569809,
      "test_blank_image_delta": 0.09699999541044235,
      "parameters": 151114
    },
    {
      "seed": 44,
      "validation_accuracy": 0.210999995470047,
      "validation_shuffled_image_accuracy": 0.10400000214576721,
      "validation_blank_image_accuracy": 0.10000000149011612,
      "validation_visual_dependency_delta": 0.10699999332427979,
      "validation_blank_image_delta": 0.11099999397993088,
      "test_accuracy": 0.1940000057220459,
      "test_shuffled_image_accuracy": 0.10999999940395355,
      "test_blank_image_accuracy": 0.10000000149011612,
      "test_visual_dependency_delta": 0.08400000631809235,
      "test_blank_image_delta": 0.09400000423192978,
      "parameters": 151114
    }
  ],
  "aggregate_metrics": {
    "validation_accuracy": {
      "mean": 0.1999999980131785,
      "std": 0.012767140278714359,
      "ci95": 0.014447379278326998,
      "n": 3
    },
    "validation_shuffled_image_accuracy": {
      "mean": 0.10333333412806193,
      "std": 0.0011547028784467755,
      "ci95": 0.001306669314702352,
      "n": 3
    },
    "validation_blank_image_accuracy": {
      "mean": 0.10000000149011612,
      "std": 0.0,
      "ci95": 0.0,
      "n": 3
    },
    "validation_visual_dependency_delta": {
      "mean": 0.09666666388511658,
      "std": 0.011676179413035684,
      "ci95": 0.013212840841356138,
      "n": 3
    },
    "validation_blank_image_delta": {
      "mean": 0.09999999652306239,
      "std": 0.012767140278714359,
      "ci95": 0.014447379278326998,
      "n": 3
    },
    "test_accuracy": {
      "mean": 0.19433333476384482,
      "std": 0.0025166086836567706,
      "ci95": 0.0028478108138702053,
      "n": 3
    },
    "test_shuffled_image_accuracy": {
      "mean": 0.10433333367109299,
      "std": 0.007371114235185654,
      "ci95": 0.008341200984307363,
      "n": 3
    },
    "test_blank_image_accuracy": {
      "mean": 0.10000000149011612,
      "std": 0.0,
      "ci95": 0.0,
      "n": 3
    },
    "test_visual_dependency_delta": {
      "mean": 0.09000000109275182,
      "std": 0.0059999972581878825,
      "ci95": 0.0067896360630175095,
      "n": 3
    },
    "test_blank_image_delta": {
      "mean": 0.09433333327372868,
      "std": 0.0025166086836567706,
      "ci95": 0.0028478108138702053,
      "n": 3
    },
    "parameters": {
      "mean": 151114.0,
      "std": 0.0,
      "ci95": 0.0,
      "n": 3
    }
  },
  "formal_comparison": true,
  "claim_policy": "multi-seed benchmark result; L1 object-QA only, not open-ended VQA",
  "evaluation_protocol": {
    "tier": "l1_public_images",
    "seeds": [
      42,
      43,
      44
    ],
    "formal_comparison": true,
    "claim_policy": "multi-seed benchmark result; L1 object-QA only, not open-ended VQA"
  },
  "provenance": {
    "artifact_path": "docs/multimodal-models/metrics/cifar10-qa-seeds42-44.json",
    "created_at": "2026-08-11T05:07:51.755553+00:00",
    "dataset_fingerprint": "official archive MD5 c58f30108f718f92721af3b95e74349a",
    "original_code_commit": "586ffe5"
  }
}