{
  "schema_version": 2,
  "method": "gemma-priority-protocol",
  "manifest_ref": "foundation-models:gemma-priority-protocol",
  "artifact_kind": "published_external_predictions_reanalysis",
  "evaluation_tier": "l1_mechanism",
  "seeds": [
    42
  ],
  "diagnostic_only": true,
  "promotion_eligible": false,
  "evolve_integrated": false,
  "model_calls": 0,
  "source_url": "https://raw.githubusercontent.com/GoogleCloudPlatform/devrel-demos/fc468f2374540870042b20db1ec7e836ed205db7/ai-ml/gemma-priority-probing/data/results/o2_results_ad_series_FINAL.json",
  "source_sha256": "744bbb4f1cda3a01c30528e1391bd6b3ec9fdaf0e75e917dfcc95ca31ca608cf",
  "upstream_revision": "fc468f2374540870042b20db1ec7e836ed205db7",
  "fixture_sha256": "38ce5312b6a16f9124b777d6e55d6f7b40c28f53110cb8da499e90ea1d023d61",
  "executed_module_sha256": "6ddb7eb4daa67f268869d69865c0ce2e91caa2452e4b1b5287ad504c7a39334a",
  "resamples": 9999,
  "primary_n": 13,
  "sensitivity_n": 14,
  "items_dropped": [
    "Spec/Brief"
  ],
  "reference_comparisons": 56,
  "reference_max_absolute_error": 0,
  "elapsed_seconds": 2.332629125099629,
  "primary_conditions": {
    "T1": {
      "token": {
        "mean": -0.21614577449285066,
        "n": 13,
        "sd": 0.39260558648916566,
        "ci95": {
          "status": "ok",
          "low": -0.43762160662297855,
          "high": -0.027934134855989848
        }
      },
      "role": {
        "mean": 0.053637025447992176,
        "n": 13,
        "sd": 0.09293731740086311,
        "ci95": {
          "status": "ok",
          "low": 0.008249617915974703,
          "high": 0.1046775196978316
        }
      },
      "order_null": {
        "mean": 0.003229102263083825,
        "n": 13,
        "sd": 0.05682123676679837,
        "ci95": {
          "status": "ok",
          "low": -0.0243833310655931,
          "high": 0.0358477061145681
        }
      },
      "serial": {
        "mean": -0.4846123479879819,
        "n": 13,
        "sd": 0.29930604655214366,
        "ci95": {
          "status": "ok",
          "low": -0.6105610799906249,
          "high": -0.28275618766935706
        }
      },
      "serial_tost": {
        "status": "ok",
        "p_value": 0.9991666956450721,
        "equivalent": false,
        "ci90": {
          "status": "ok",
          "low": -0.5916090951395521,
          "high": -0.31827997158141785
        },
        "bound": 0.15
      }
    },
    "T1c": {
      "token": {
        "mean": -0.14221848432834333,
        "n": 13,
        "sd": 0.4019613966123722,
        "ci95": {
          "status": "ok",
          "low": -0.35981025508800113,
          "high": 0.05762722060067935
        }
      },
      "role": {
        "mean": 0.007959471299098088,
        "n": 13,
        "sd": 0.12326620282453528,
        "ci95": {
          "status": "ok",
          "low": -0.04495927922465028,
          "high": 0.08828853005012784
        }
      },
      "order_null": {
        "mean": 0.05627135588572575,
        "n": 13,
        "sd": 0.1434053400773278,
        "ci95": {
          "status": "ok",
          "low": -0.025396127871829377,
          "high": 0.12521651590098004
        }
      },
      "serial": {
        "mean": -0.8841167184022757,
        "n": 13,
        "sd": 0.35590000416333223,
        "ci95": {
          "status": "ok",
          "low": -1.074233508031551,
          "high": -0.707635009667032
        }
      },
      "serial_tost": {
        "status": "ok",
        "p_value": 0.9999960650464078,
        "equivalent": false,
        "ci90": {
          "status": "ok",
          "low": -1.0436552034593845,
          "high": -0.7350980856879207
        },
        "bound": 0.15
      }
    },
    "T2": {
      "token": {
        "mean": -0.1305578752205922,
        "n": 13,
        "sd": 0.33153330272745085,
        "ci95": {
          "status": "ok",
          "low": -0.3092233119419867,
          "high": 0.03258819980272361
        }
      },
      "role": {
        "mean": 0.03939377459195944,
        "n": 13,
        "sd": 0.0673154503809881,
        "ci95": {
          "status": "ok",
          "low": 0.005764743195649605,
          "high": 0.0758485254070245
        }
      },
      "order_null": {
        "mean": 0.05315932746116932,
        "n": 13,
        "sd": 0.07647996309560798,
        "ci95": {
          "status": "ok",
          "low": 0.019545372955437287,
          "high": 0.10038762940707968
        }
      },
      "serial": {
        "mean": -0.37673629361849564,
        "n": 13,
        "sd": 0.25961400245590976,
        "ci95": {
          "status": "ok",
          "low": -0.4959174300957909,
          "high": -0.22729703848301158
        }
      },
      "serial_tost": {
        "status": "ok",
        "p_value": 0.9958042967291285,
        "equivalent": false,
        "ci90": {
          "status": "ok",
          "low": -0.47841855692705415,
          "high": -0.2517363466072862
        },
        "bound": 0.15
      }
    },
    "T2c": {
      "token": {
        "mean": -0.015148918216045085,
        "n": 13,
        "sd": 0.4127793433217034,
        "ci95": {
          "status": "ok",
          "low": -0.2225604975637494,
          "high": 0.20943697694933108
        }
      },
      "role": {
        "mean": -0.06574823191532722,
        "n": 13,
        "sd": 0.11900874064907592,
        "ci95": {
          "status": "ok",
          "low": -0.1252309783339666,
          "high": -0.0018561399576353527
        }
      },
      "order_null": {
        "mean": 0.06966497233280769,
        "n": 13,
        "sd": 0.1466808742513027,
        "ci95": {
          "status": "ok",
          "low": -0.002447736370794328,
          "high": 0.14829156686443362
        }
      },
      "serial": {
        "mean": -0.6345615215026416,
        "n": 13,
        "sd": 0.42006220446889153,
        "ci95": {
          "status": "ok",
          "low": -0.9018693489890477,
          "high": -0.45239388100081646
        }
      },
      "serial_tost": {
        "status": "ok",
        "p_value": 0.9993376269400913,
        "equivalent": false,
        "ci90": {
          "status": "ok",
          "low": -0.8540124747025845,
          "high": -0.47776050245859664
        },
        "bound": 0.15
      }
    },
    "T3": {
      "token": {
        "mean": -0.13936613843991205,
        "n": 13,
        "sd": 0.35530802296037467,
        "ci95": {
          "status": "ok",
          "low": -0.32877250240044126,
          "high": 0.041036368304143464
        }
      },
      "role": {
        "mean": -0.15932691326508155,
        "n": 13,
        "sd": 0.09825414355367365,
        "ci95": {
          "status": "ok",
          "low": -0.20818304825941614,
          "high": -0.10667709043501225
        }
      },
      "order_null": {
        "mean": 0.019720817987735454,
        "n": 13,
        "sd": 0.11555991182820062,
        "ci95": {
          "status": "ok",
          "low": -0.048516096710127465,
          "high": 0.07303830429037592
        }
      },
      "serial": {
        "mean": -0.6787652946435488,
        "n": 13,
        "sd": 0.3157272130323785,
        "ci95": {
          "status": "ok",
          "low": -0.8476333841812588,
          "high": -0.5203439557275888
        }
      },
      "serial_tost": {
        "status": "ok",
        "p_value": 0.9999706923712403,
        "equivalent": false,
        "ci90": {
          "status": "ok",
          "low": -0.8224031137275772,
          "high": -0.5428157314825863
        },
        "bound": 0.15
      }
    },
    "T3c": {
      "token": {
        "mean": -0.12362858194571275,
        "n": 13,
        "sd": 0.29279113722071815,
        "ci95": {
          "status": "ok",
          "low": -0.30961158832810987,
          "high": 0.003078930538086379
        }
      },
      "role": {
        "mean": 0.07160389652618995,
        "n": 13,
        "sd": 0.08970602454514869,
        "ci95": {
          "status": "ok",
          "low": 0.02390546733317603,
          "high": 0.11741758052421529
        }
      },
      "order_null": {
        "mean": -0.01646164059638977,
        "n": 13,
        "sd": 0.16055328677121763,
        "ci95": {
          "status": "ok",
          "low": -0.11579823283117327,
          "high": 0.05503585907316404
        }
      },
      "serial": {
        "mean": -0.5691753969742701,
        "n": 13,
        "sd": 0.29551803626353457,
        "ci95": {
          "status": "ok",
          "low": -0.7385312294291762,
          "high": -0.4289760874545933
        }
      },
      "serial_tost": {
        "status": "ok",
        "p_value": 0.9998721616137737,
        "equivalent": false,
        "ci90": {
          "status": "ok",
          "low": -0.7130193202788044,
          "high": -0.4515855992008424
        },
        "bound": 0.15
      }
    },
    "T4": {
      "token": {
        "mean": -0.08018898963928223,
        "n": 13,
        "sd": 0.38556356933645924,
        "ci95": {
          "status": "ok",
          "low": -0.2952204992119011,
          "high": 0.10982774895149741
        }
      },
      "role": {
        "mean": -0.01773106364103464,
        "n": 13,
        "sd": 0.10917638152475917,
        "ci95": {
          "status": "ok",
          "low": -0.064826413421808,
          "high": 0.05210759821464582
        }
      },
      "order_null": {
        "mean": -0.010260971692892222,
        "n": 13,
        "sd": 0.17214262608095543,
        "ci95": {
          "status": "ok",
          "low": -0.11375496668199316,
          "high": 0.0696521724953309
        }
      },
      "serial": {
        "mean": -0.17188627674029425,
        "n": 13,
        "sd": 0.20447288163322752,
        "ci95": {
          "status": "ok",
          "low": -0.2777446228973527,
          "high": -0.06616205651927806
        }
      },
      "serial_tost": {
        "status": "ok",
        "p_value": 0.6468480762393086,
        "equivalent": false,
        "ci90": {
          "status": "ok",
          "low": -0.262275184539765,
          "high": -0.08171847435921552
        },
        "bound": 0.15
      }
    },
    "T4c": {
      "token": {
        "mean": -0.12774086457032424,
        "n": 13,
        "sd": 0.3402037707560368,
        "ci95": {
          "status": "ok",
          "low": -0.328931395275914,
          "high": 0.027704523056356557
        }
      },
      "role": {
        "mean": 0.0053238891638242285,
        "n": 13,
        "sd": 0.11330734265536613,
        "ci95": {
          "status": "ok",
          "low": -0.056143093657163294,
          "high": 0.0629379280067845
        }
      },
      "order_null": {
        "mean": -0.017249928070948675,
        "n": 13,
        "sd": 0.09715280134064753,
        "ci95": {
          "status": "ok",
          "low": -0.07348824329368125,
          "high": 0.028349430946974875
        }
      },
      "serial": {
        "mean": -0.48162200588446397,
        "n": 13,
        "sd": 0.25116910234463685,
        "ci95": {
          "status": "ok",
          "low": -0.6027123672558601,
          "high": -0.3446860156716285
        }
      },
      "serial_tost": {
        "status": "ok",
        "p_value": 0.999768119452754,
        "equivalent": false,
        "ci90": {
          "status": "ok",
          "low": -0.586287299288388,
          "high": -0.36629202873188
        },
        "bound": 0.15
      }
    },
    "T5": {
      "token": {
        "mean": -0.14423120938814604,
        "n": 13,
        "sd": 0.3545758436618192,
        "ci95": {
          "status": "ok",
          "low": -0.3628055301112203,
          "high": 0.01183862416089626
        }
      },
      "role": {
        "mean": 0.515922317138085,
        "n": 13,
        "sd": 0.34582255446988674,
        "ci95": {
          "status": "ok",
          "low": 0.2763711138111145,
          "high": 0.659924376771021
        }
      },
      "order_null": {
        "mean": 0.011722115369943472,
        "n": 13,
        "sd": 0.2092876478768675,
        "ci95": {
          "status": "ok",
          "low": -0.0824125102470546,
          "high": 0.13960688633480983
        }
      },
      "serial": {
        "mean": -0.2047865207378681,
        "n": 13,
        "sd": 0.22712483165438813,
        "ci95": {
          "status": "ok",
          "low": -0.308243326722201,
          "high": -0.06780218419648561
        }
      },
      "serial_tost": {
        "status": "ok",
        "p_value": 0.7992429032159659,
        "equivalent": false,
        "ci90": {
          "status": "ok",
          "low": -0.2927937390856146,
          "high": -0.09102467672543668
        },
        "bound": 0.15
      }
    },
    "T5b": {
      "token": {
        "mean": 0.0031024492703951322,
        "n": 13,
        "sd": 0.28961938922319447,
        "ci95": {
          "status": "ok",
          "low": -0.13534267127818578,
          "high": 0.16457210783580056
        }
      },
      "role": {
        "mean": 1.054441974713252,
        "n": 13,
        "sd": 0.32993565796759367,
        "ci95": {
          "status": "ok",
          "low": 0.9069644099530213,
          "high": 1.2576188751984443
        }
      },
      "order_null": {
        "mean": 0.01099850581242488,
        "n": 13,
        "sd": 0.14011564686185793,
        "ci95": {
          "status": "ok",
          "low": -0.058228599074954104,
          "high": 0.08737334939895552
        }
      },
      "serial": {
        "mean": -0.26548606615800124,
        "n": 13,
        "sd": 0.2716472842038903,
        "ci95": {
          "status": "ok",
          "low": -0.420917292624819,
          "high": -0.13697238486280947
        }
      },
      "serial_tost": {
        "status": "ok",
        "p_value": 0.9243765512043223,
        "equivalent": false,
        "ci90": {
          "status": "ok",
          "low": -0.39359232441075476,
          "high": -0.1585208976260169
        },
        "bound": 0.15
      }
    }
  },
  "primary_contrasts": {
    "A1": {
      "mean": -0.3995043704142937,
      "n": 13,
      "sd": 0.324447357875717,
      "ci95": {
        "status": "ok",
        "low": -0.5776041162020543,
        "high": -0.23743063623821492
      }
    },
    "A2": {
      "mean": 0.10958989766927865,
      "n": 13,
      "sd": 0.350693897624307,
      "ci95": {
        "status": "ok",
        "low": -0.08084770179657114,
        "high": 0.28683840630842905
      }
    },
    "A3": {
      "mean": -0.14495723637250754,
      "n": 13,
      "sd": 0.2874796244894381,
      "ci95": {
        "status": "ok",
        "low": -0.2904003219518333,
        "high": 0.009712109910773897
      }
    },
    "A4": {
      "mean": -0.5090942680835724,
      "n": 13,
      "sd": 0.35486118094343566,
      "ci95": {
        "status": "ok",
        "low": -0.6871297301859844,
        "high": -0.3164169542968323
      }
    },
    "B1": {
      "mean": 0.1078760543694863,
      "n": 13,
      "sd": 0.2600994232161995,
      "ci95": {
        "status": "ok",
        "low": -0.025413694414350717,
        "high": 0.24493829591147026
      }
    },
    "B2": {
      "mean": -0.25782522788414586,
      "n": 13,
      "sd": 0.33682107678487794,
      "ci95": {
        "status": "ok",
        "low": -0.44924338624933113,
        "high": -0.09642693974438073
      }
    },
    "B3": {
      "mean": -0.14994917351465958,
      "n": 13,
      "sd": 0.4881867106809273,
      "ci95": {
        "status": "ok",
        "low": -0.4308240059736667,
        "high": 0.0786171423767411
      }
    },
    "B4": {
      "mean": 0.24955519689963415,
      "n": 13,
      "sd": 0.49335463827920956,
      "ci95": {
        "status": "ok",
        "low": 0.009775253136105697,
        "high": 0.5215603936121426
      }
    },
    "C1": {
      "mean": 0.15293951561817756,
      "n": 13,
      "sd": 0.34504871753680577,
      "ci95": {
        "status": "ok",
        "low": -0.01106837775776555,
        "high": 0.35218178490745966
      }
    },
    "C2": {
      "mean": 0.3149413214280055,
      "n": 13,
      "sd": 0.232414973392756,
      "ci95": {
        "status": "ok",
        "low": 0.17201216132445718,
        "high": 0.419910109988629
      }
    },
    "C3": {
      "mean": -0.16200180580982795,
      "n": 13,
      "sd": 0.25573874553353376,
      "ci95": {
        "status": "ok",
        "low": -0.3025986816939087,
        "high": -0.0339958302992406
      }
    },
    "C4": {
      "mean": -0.19415294665556687,
      "n": 13,
      "sd": 0.3589381542865871,
      "ci95": {
        "status": "ok",
        "low": -0.36730936349893023,
        "high": 0.014794112890013622
      }
    },
    "C5": {
      "mean": 0.34709246227374446,
      "n": 13,
      "sd": 0.3474356744405964,
      "ci95": {
        "status": "ok",
        "low": 0.15983121440359255,
        "high": 0.5192758386083225
      }
    },
    "C6": {
      "mean": 0.20485001687820142,
      "n": 13,
      "sd": 0.20579768622676733,
      "ci95": {
        "status": "ok",
        "low": 0.09790618563059171,
        "high": 0.31344647481377474
      }
    },
    "D1": {
      "mean": 0.7740303965715262,
      "n": 13,
      "sd": 0.36967108595666975,
      "ci95": {
        "status": "ok",
        "low": 0.584544047879219,
        "high": 0.9664333629947122
      }
    },
    "D2": {
      "mean": 0.3161560297012329,
      "n": 13,
      "sd": 0.236201586510632,
      "ci95": {
        "status": "ok",
        "low": 0.188015451352152,
        "high": 0.43410878101719585
      }
    },
    "D3": {
      "mean": 0.5385196575751672,
      "n": 13,
      "sd": 0.4847666579698041,
      "ci95": {
        "status": "ok",
        "low": 0.32959642135179723,
        "high": 0.843157624417122
      }
    },
    "E1": {
      "mean": 0.21613593972646272,
      "n": 13,
      "sd": 0.3412738356255258,
      "ci95": {
        "status": "ok",
        "low": -0.0037767418926592366,
        "high": 0.36421506981047846
      }
    },
    "E2": {
      "mean": -0.03290024399757385,
      "n": 13,
      "sd": 0.19506133351178523,
      "ci95": {
        "status": "ok",
        "low": -0.12359628466582007,
        "high": 0.08282160694978884
      }
    }
  },
  "sensitivity_contrasts": {
    "A1": {
      "mean": -0.4073023136172976,
      "n": 14,
      "sd": 0.3130814710265963,
      "ci95": {
        "status": "ok",
        "low": -0.573107550767587,
        "high": -0.25064022314163714
      }
    },
    "A2": {
      "mean": 0.07431230800492424,
      "n": 14,
      "sd": 0.36186855183477573,
      "ci95": {
        "status": "ok",
        "low": -0.11222086637300788,
        "high": 0.25675370798340963
      }
    },
    "A3": {
      "mean": -0.16649500280618668,
      "n": 14,
      "sd": 0.28771777486550837,
      "ci95": {
        "status": "ok",
        "low": -0.3062317872618551,
        "high": -0.011151416744707921
      }
    },
    "A4": {
      "mean": -0.4816146216222218,
      "n": 14,
      "sd": 0.356106221119644,
      "ci95": {
        "status": "ok",
        "low": -0.658249266194236,
        "high": -0.2989602530326534
      }
    },
    "B1": {
      "mean": 0.11042929547173637,
      "n": 14,
      "sd": 0.25007798549679655,
      "ci95": {
        "status": "ok",
        "low": -0.018121366939784084,
        "high": 0.23464955110249916
      }
    },
    "B2": {
      "mean": -0.2752074162874903,
      "n": 14,
      "sd": 0.3300781748208422,
      "ci95": {
        "status": "ok",
        "low": -0.45022137083078406,
        "high": -0.11908288157165364
      }
    },
    "B3": {
      "mean": -0.16477812081575394,
      "n": 14,
      "sd": 0.4723050311001137,
      "ci95": {
        "status": "ok",
        "low": -0.4293432849227392,
        "high": 0.05179410336978255
      }
    },
    "B4": {
      "mean": 0.24252419280154364,
      "n": 14,
      "sd": 0.4747292963630388,
      "ci95": {
        "status": "ok",
        "low": 0.022250252971302713,
        "high": 0.5016775859056501
      }
    },
    "C1": {
      "mean": 0.12256017220871789,
      "n": 14,
      "sd": 0.3504581649317944,
      "ci95": {
        "status": "ok",
        "low": -0.03264052279519629,
        "high": 0.32163281177429437
      }
    },
    "C2": {
      "mean": 0.29572826836790356,
      "n": 14,
      "sd": 0.23458381715479715,
      "ci95": {
        "status": "ok",
        "low": 0.1662986981462766,
        "high": 0.4033418577747905
      }
    },
    "C3": {
      "mean": -0.17316809615918569,
      "n": 14,
      "sd": 0.24923275048945837,
      "ci95": {
        "status": "ok",
        "low": -0.3005144184046844,
        "high": -0.0481484613096407
      }
    },
    "C4": {
      "mean": -0.18588635325431824,
      "n": 14,
      "sd": 0.34624096081819034,
      "ci95": {
        "status": "ok",
        "low": -0.3486939338360071,
        "high": 0.0035032561750097295
      }
    },
    "C5": {
      "mean": 0.30844652546303614,
      "n": 14,
      "sd": 0.36377900192157736,
      "ci95": {
        "status": "ok",
        "low": 0.12487073180753926,
        "high": 0.49188866138289444
      }
    },
    "C6": {
      "mean": 0.19778337436062948,
      "n": 14,
      "sd": 0.19948411645141884,
      "ci95": {
        "status": "ok",
        "low": 0.10196037739043669,
        "high": 0.29788302289800167
      }
    },
    "D1": {
      "mean": 0.8061408954007285,
      "n": 14,
      "sd": 0.3749397750705653,
      "ci95": {
        "status": "ok",
        "low": 0.6160037711353865,
        "high": 0.9908023051867844
      }
    },
    "D2": {
      "mean": 0.31736286197389874,
      "n": 14,
      "sd": 0.22698006460651768,
      "ci95": {
        "status": "ok",
        "low": 0.19867689986090004,
        "high": 0.42824187147322107
      }
    },
    "D3": {
      "mean": 0.5827449836901256,
      "n": 14,
      "sd": 0.49427138416724864,
      "ci95": {
        "status": "ok",
        "low": 0.36593045264482726,
        "high": 0.8741900485524586
      }
    },
    "E1": {
      "mean": 0.23736193776130676,
      "n": 14,
      "sd": 0.3373668164840287,
      "ci95": {
        "status": "ok",
        "low": 0.02467372655632477,
        "high": 0.37479768658459656
      }
    },
    "E2": {
      "mean": -0.03762277960777283,
      "n": 14,
      "sd": 0.18824004975808817,
      "ci95": {
        "status": "ok",
        "low": -0.11904408996095511,
        "high": 0.07179065441282471
      }
    }
  },
  "claim": "Arithmetic and BCa/TOST reanalysis of fixed author observations; no newly evaluated model or independent replication of the source predictions",
  "a4_reference": "Derived from observed cells; no A4 field in released author FINAL.json",
  "evaluation_protocol": {
    "tier": "l1_mechanism",
    "seeds": [
      42
    ],
    "formal_comparison": false,
    "claim_policy": "fixed published observations reanalysis; no independent model inference or capability comparison"
  },
  "provenance": {
    "artifact_path": "docs/foundation-models/2609.30716-gemma-priority-protocol/metrics/official-log-reanalysis.json",
    "dataset_fingerprint": "sha256:38ce5312b6a16f9124b777d6e55d6f7b40c28f53110cb8da499e90ea1d023d61",
    "accelerator": "CPU",
    "source_observations": "fixed author FINAL.json, not newly inferred predictions"
  }
}