{
  "schema_version": 2,
  "seeds_by_adapter": {
    "recevolve": [
      42,
      43,
      44
    ]
  },
  "budget": "paper-specific",
  "papers": {
    "recevolve": {
      "manifest": {
        "adapter_key": "recevolve",
        "arxiv_id": "2609.01622",
        "title": "RecEvolve: A Knowledge-Driven Autonomous Agent System for Recommender Systems",
        "paper_url": "https://arxiv.org/abs/2609.01622",
        "track": "recommendation",
        "organization": "Google",
        "published": "2026-07-20",
        "code_url": null,
        "topics": [
          "auto-research",
          "agent",
          "two-tower",
          "reward-hacking"
        ],
        "local_code_dir": "src/auto_research/reproductions/recevolve",
        "fidelity": "core_mechanism",
        "evaluation_tier": "l2_public_dataset",
        "datasets": [
          "MovieLens 100K"
        ],
        "baseline": "fixed Two-Tower-style similarity",
        "metrics": [
          "hit_at_10",
          "ndcg_at_10",
          "rollbacks",
          "reward hacks blocked"
        ],
        "default_seeds": [
          42,
          43,
          44
        ],
        "budget": "adapter-defined fixed budget",
        "device_capabilities": [
          "cpu"
        ],
        "requires_gpu_validation": false,
        "gpu_validation_artifact": null,
        "online_evidence": [
          {
            "product": "Google production Two-Tower retrieval",
            "metric": "user satisfaction",
            "lift_percent": 3.77,
            "traffic": "live production A/B champion versus production baseline",
            "source_url": "https://arxiv.org/pdf/2609.01622v1",
            "source_location": "Section 5.7, Table 2",
            "retrieved_at": "2026-09-06"
          }
        ],
        "selection_exception": null,
        "evolve_operators": [
          "controller:recevolve-knowledge-vcs-loop"
        ]
      },
      "seed_results": [
        {
          "paper": {
            "arxiv_id": "2609.01622",
            "title": "RecEvolve: A Knowledge-Driven Autonomous Agent System for Recommender Systems",
            "url": "https://arxiv.org/abs/2609.01622",
            "organization": "Google"
          },
          "dataset": {
            "name": "MovieLens 100K",
            "users": 220,
            "items": 360
          },
          "setup": {
            "adapter": "recevolve",
            "same_split_and_candidates": true
          },
          "baseline": {
            "name": "fixed Two-Tower-style similarity",
            "hit_at_10": 0.10909090909090909,
            "ndcg_at_10": 0.05400546778896325,
            "fresh_hit_at_10": 0.031746031746031744,
            "head_share_at_10": 0.23772727272727273
          },
          "method": {
            "name": "validation-selected RecEvolve champion",
            "hit_at_10": 0.10909090909090909,
            "ndcg_at_10": 0.054310517127949154,
            "fresh_hit_at_10": 0.031746031746031744,
            "head_share_at_10": 0.2913636363636364
          },
          "relative": {
            "hit_at_10_percent": 0.0,
            "ndcg_at_10_percent": 0.5648489893244559,
            "fresh_hit_at_10_percent": 0.0,
            "head_share_at_10_percent": 22.5621414913958
          },
          "stages": {
            "experiments": 6,
            "isolated_trials": 5,
            "rollbacks": 4,
            "reward_hacks_blocked": 1,
            "champion": {
              "temperature": 0.8,
              "watch_power": 0.5,
              "cross_strength": 0.5,
              "cosine_decay": false
            }
          },
          "paper_results": {
            "offline_ndcg50_gain_percent": 19.9,
            "online_user_satisfaction_percent": 3.77,
            "online_unique_content_percent": 7.44,
            "autonomous_runs": 41
          },
          "scope": "公开 MovieLens 上执行提案、critic gate、隔离评价、冠军继承、回滚和 reward-hack 拒绝；不复刻 Google 私有 Two-Tower、TPU 集群和线上流量。",
          "runtime": {
            "requested_device": "auto",
            "cpu_threads": null,
            "platform": "Darwin arm64",
            "resolved_device": "cpu",
            "torch_version": "2.13.0",
            "accelerator": "arm"
          },
          "seed": 42,
          "schema_version": 2,
          "manifest": {
            "adapter_key": "recevolve",
            "arxiv_id": "2609.01622",
            "title": "RecEvolve: A Knowledge-Driven Autonomous Agent System for Recommender Systems",
            "paper_url": "https://arxiv.org/abs/2609.01622",
            "track": "recommendation",
            "organization": "Google",
            "published": "2026-07-20",
            "code_url": null,
            "topics": [
              "auto-research",
              "agent",
              "two-tower",
              "reward-hacking"
            ],
            "local_code_dir": "src/auto_research/reproductions/recevolve",
            "fidelity": "core_mechanism",
            "evaluation_tier": "l2_public_dataset",
            "datasets": [
              "MovieLens 100K"
            ],
            "baseline": "fixed Two-Tower-style similarity",
            "metrics": [
              "hit_at_10",
              "ndcg_at_10",
              "rollbacks",
              "reward hacks blocked"
            ],
            "default_seeds": [
              42,
              43,
              44
            ],
            "budget": "adapter-defined fixed budget",
            "device_capabilities": [
              "cpu"
            ],
            "requires_gpu_validation": false,
            "gpu_validation_artifact": null,
            "online_evidence": [
              {
                "product": "Google production Two-Tower retrieval",
                "metric": "user satisfaction",
                "lift_percent": 3.77,
                "traffic": "live production A/B champion versus production baseline",
                "source_url": "https://arxiv.org/pdf/2609.01622v1",
                "source_location": "Section 5.7, Table 2",
                "retrieved_at": "2026-09-06"
              }
            ],
            "selection_exception": null,
            "evolve_operators": [
              "controller:recevolve-knowledge-vcs-loop"
            ]
          },
          "provenance": {
            "created_at": "2026-09-05T19:38:42.771791+00:00",
            "code_commit": "6d97d3b2028b651e46229d2596e866d2a6f567c9",
            "python": "3.12.9",
            "platform": "macOS-26.5.2-arm64-arm-64bit",
            "dataset_dir": "/Users/bytedance/Documents/git_daiwk/auto-research/data",
            "dataset_fingerprint": "e66c261317a7b3179720ef3e3f5f79d7a204b135d1f217e2911b153ad2ce50a0",
            "packages": {
              "auto-research": "0.1.0",
              "numpy": "2.5.2",
              "torch": "2.13.0",
              "transformers": "5.14.1"
            }
          },
          "evaluation_protocol": {
            "tier": "l2_public_dataset",
            "tier_label": "L2 公开数据集训练",
            "seeds": [
              42
            ],
            "budget": "paper-specific",
            "formal_comparison": false,
            "claim_policy": "single/few-seed smoke result; do not claim a stable improvement"
          }
        },
        {
          "paper": {
            "arxiv_id": "2609.01622",
            "title": "RecEvolve: A Knowledge-Driven Autonomous Agent System for Recommender Systems",
            "url": "https://arxiv.org/abs/2609.01622",
            "organization": "Google"
          },
          "dataset": {
            "name": "MovieLens 100K",
            "users": 220,
            "items": 360
          },
          "setup": {
            "adapter": "recevolve",
            "same_split_and_candidates": true
          },
          "baseline": {
            "name": "fixed Two-Tower-style similarity",
            "hit_at_10": 0.10909090909090909,
            "ndcg_at_10": 0.05400546778896325,
            "fresh_hit_at_10": 0.031746031746031744,
            "head_share_at_10": 0.23772727272727273
          },
          "method": {
            "name": "validation-selected RecEvolve champion",
            "hit_at_10": 0.10909090909090909,
            "ndcg_at_10": 0.054310517127949154,
            "fresh_hit_at_10": 0.031746031746031744,
            "head_share_at_10": 0.2913636363636364
          },
          "relative": {
            "hit_at_10_percent": 0.0,
            "ndcg_at_10_percent": 0.5648489893244559,
            "fresh_hit_at_10_percent": 0.0,
            "head_share_at_10_percent": 22.5621414913958
          },
          "stages": {
            "experiments": 6,
            "isolated_trials": 5,
            "rollbacks": 4,
            "reward_hacks_blocked": 1,
            "champion": {
              "temperature": 0.8,
              "watch_power": 0.5,
              "cross_strength": 0.5,
              "cosine_decay": false
            }
          },
          "paper_results": {
            "offline_ndcg50_gain_percent": 19.9,
            "online_user_satisfaction_percent": 3.77,
            "online_unique_content_percent": 7.44,
            "autonomous_runs": 41
          },
          "scope": "公开 MovieLens 上执行提案、critic gate、隔离评价、冠军继承、回滚和 reward-hack 拒绝；不复刻 Google 私有 Two-Tower、TPU 集群和线上流量。",
          "runtime": {
            "requested_device": "auto",
            "cpu_threads": null,
            "platform": "Darwin arm64",
            "resolved_device": "cpu",
            "torch_version": "2.13.0",
            "accelerator": "arm"
          },
          "seed": 43,
          "schema_version": 2,
          "manifest": {
            "adapter_key": "recevolve",
            "arxiv_id": "2609.01622",
            "title": "RecEvolve: A Knowledge-Driven Autonomous Agent System for Recommender Systems",
            "paper_url": "https://arxiv.org/abs/2609.01622",
            "track": "recommendation",
            "organization": "Google",
            "published": "2026-07-20",
            "code_url": null,
            "topics": [
              "auto-research",
              "agent",
              "two-tower",
              "reward-hacking"
            ],
            "local_code_dir": "src/auto_research/reproductions/recevolve",
            "fidelity": "core_mechanism",
            "evaluation_tier": "l2_public_dataset",
            "datasets": [
              "MovieLens 100K"
            ],
            "baseline": "fixed Two-Tower-style similarity",
            "metrics": [
              "hit_at_10",
              "ndcg_at_10",
              "rollbacks",
              "reward hacks blocked"
            ],
            "default_seeds": [
              42,
              43,
              44
            ],
            "budget": "adapter-defined fixed budget",
            "device_capabilities": [
              "cpu"
            ],
            "requires_gpu_validation": false,
            "gpu_validation_artifact": null,
            "online_evidence": [
              {
                "product": "Google production Two-Tower retrieval",
                "metric": "user satisfaction",
                "lift_percent": 3.77,
                "traffic": "live production A/B champion versus production baseline",
                "source_url": "https://arxiv.org/pdf/2609.01622v1",
                "source_location": "Section 5.7, Table 2",
                "retrieved_at": "2026-09-06"
              }
            ],
            "selection_exception": null,
            "evolve_operators": [
              "controller:recevolve-knowledge-vcs-loop"
            ]
          },
          "provenance": {
            "created_at": "2026-09-05T19:38:42.993914+00:00",
            "code_commit": "6d97d3b2028b651e46229d2596e866d2a6f567c9",
            "python": "3.12.9",
            "platform": "macOS-26.5.2-arm64-arm-64bit",
            "dataset_dir": "/Users/bytedance/Documents/git_daiwk/auto-research/data",
            "dataset_fingerprint": "e66c261317a7b3179720ef3e3f5f79d7a204b135d1f217e2911b153ad2ce50a0",
            "packages": {
              "auto-research": "0.1.0",
              "numpy": "2.5.2",
              "torch": "2.13.0",
              "transformers": "5.14.1"
            }
          },
          "evaluation_protocol": {
            "tier": "l2_public_dataset",
            "tier_label": "L2 公开数据集训练",
            "seeds": [
              43
            ],
            "budget": "paper-specific",
            "formal_comparison": false,
            "claim_policy": "single/few-seed smoke result; do not claim a stable improvement"
          }
        },
        {
          "paper": {
            "arxiv_id": "2609.01622",
            "title": "RecEvolve: A Knowledge-Driven Autonomous Agent System for Recommender Systems",
            "url": "https://arxiv.org/abs/2609.01622",
            "organization": "Google"
          },
          "dataset": {
            "name": "MovieLens 100K",
            "users": 220,
            "items": 360
          },
          "setup": {
            "adapter": "recevolve",
            "same_split_and_candidates": true
          },
          "baseline": {
            "name": "fixed Two-Tower-style similarity",
            "hit_at_10": 0.10909090909090909,
            "ndcg_at_10": 0.05400546778896325,
            "fresh_hit_at_10": 0.031746031746031744,
            "head_share_at_10": 0.23772727272727273
          },
          "method": {
            "name": "validation-selected RecEvolve champion",
            "hit_at_10": 0.10909090909090909,
            "ndcg_at_10": 0.054310517127949154,
            "fresh_hit_at_10": 0.031746031746031744,
            "head_share_at_10": 0.2913636363636364
          },
          "relative": {
            "hit_at_10_percent": 0.0,
            "ndcg_at_10_percent": 0.5648489893244559,
            "fresh_hit_at_10_percent": 0.0,
            "head_share_at_10_percent": 22.5621414913958
          },
          "stages": {
            "experiments": 6,
            "isolated_trials": 5,
            "rollbacks": 4,
            "reward_hacks_blocked": 1,
            "champion": {
              "temperature": 0.8,
              "watch_power": 0.5,
              "cross_strength": 0.5,
              "cosine_decay": false
            }
          },
          "paper_results": {
            "offline_ndcg50_gain_percent": 19.9,
            "online_user_satisfaction_percent": 3.77,
            "online_unique_content_percent": 7.44,
            "autonomous_runs": 41
          },
          "scope": "公开 MovieLens 上执行提案、critic gate、隔离评价、冠军继承、回滚和 reward-hack 拒绝；不复刻 Google 私有 Two-Tower、TPU 集群和线上流量。",
          "runtime": {
            "requested_device": "auto",
            "cpu_threads": null,
            "platform": "Darwin arm64",
            "resolved_device": "cpu",
            "torch_version": "2.13.0",
            "accelerator": "arm"
          },
          "seed": 44,
          "schema_version": 2,
          "manifest": {
            "adapter_key": "recevolve",
            "arxiv_id": "2609.01622",
            "title": "RecEvolve: A Knowledge-Driven Autonomous Agent System for Recommender Systems",
            "paper_url": "https://arxiv.org/abs/2609.01622",
            "track": "recommendation",
            "organization": "Google",
            "published": "2026-07-20",
            "code_url": null,
            "topics": [
              "auto-research",
              "agent",
              "two-tower",
              "reward-hacking"
            ],
            "local_code_dir": "src/auto_research/reproductions/recevolve",
            "fidelity": "core_mechanism",
            "evaluation_tier": "l2_public_dataset",
            "datasets": [
              "MovieLens 100K"
            ],
            "baseline": "fixed Two-Tower-style similarity",
            "metrics": [
              "hit_at_10",
              "ndcg_at_10",
              "rollbacks",
              "reward hacks blocked"
            ],
            "default_seeds": [
              42,
              43,
              44
            ],
            "budget": "adapter-defined fixed budget",
            "device_capabilities": [
              "cpu"
            ],
            "requires_gpu_validation": false,
            "gpu_validation_artifact": null,
            "online_evidence": [
              {
                "product": "Google production Two-Tower retrieval",
                "metric": "user satisfaction",
                "lift_percent": 3.77,
                "traffic": "live production A/B champion versus production baseline",
                "source_url": "https://arxiv.org/pdf/2609.01622v1",
                "source_location": "Section 5.7, Table 2",
                "retrieved_at": "2026-09-06"
              }
            ],
            "selection_exception": null,
            "evolve_operators": [
              "controller:recevolve-knowledge-vcs-loop"
            ]
          },
          "provenance": {
            "created_at": "2026-09-05T19:38:43.140515+00:00",
            "code_commit": "6d97d3b2028b651e46229d2596e866d2a6f567c9",
            "python": "3.12.9",
            "platform": "macOS-26.5.2-arm64-arm-64bit",
            "dataset_dir": "/Users/bytedance/Documents/git_daiwk/auto-research/data",
            "dataset_fingerprint": "e66c261317a7b3179720ef3e3f5f79d7a204b135d1f217e2911b153ad2ce50a0",
            "packages": {
              "auto-research": "0.1.0",
              "numpy": "2.5.2",
              "torch": "2.13.0",
              "transformers": "5.14.1"
            }
          },
          "evaluation_protocol": {
            "tier": "l2_public_dataset",
            "tier_label": "L2 公开数据集训练",
            "seeds": [
              44
            ],
            "budget": "paper-specific",
            "formal_comparison": false,
            "claim_policy": "single/few-seed smoke result; do not claim a stable improvement"
          }
        }
      ],
      "aggregate_metrics": {
        "dataset.users": {
          "mean": 220.0,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "dataset.items": {
          "mean": 360.0,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "baseline.hit_at_10": {
          "mean": 0.10909090909090909,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "baseline.ndcg_at_10": {
          "mean": 0.05400546778896325,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "baseline.fresh_hit_at_10": {
          "mean": 0.031746031746031744,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "baseline.head_share_at_10": {
          "mean": 0.2377272727272727,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "method.hit_at_10": {
          "mean": 0.10909090909090909,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "method.ndcg_at_10": {
          "mean": 0.054310517127949154,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "method.fresh_hit_at_10": {
          "mean": 0.031746031746031744,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "method.head_share_at_10": {
          "mean": 0.2913636363636364,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "relative.hit_at_10_percent": {
          "mean": 0.0,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "relative.ndcg_at_10_percent": {
          "mean": 0.5648489893244559,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "relative.fresh_hit_at_10_percent": {
          "mean": 0.0,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "relative.head_share_at_10_percent": {
          "mean": 22.562141491395796,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "stages.experiments": {
          "mean": 6.0,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "stages.isolated_trials": {
          "mean": 5.0,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "stages.rollbacks": {
          "mean": 4.0,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "stages.reward_hacks_blocked": {
          "mean": 1.0,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "stages.champion.temperature": {
          "mean": 0.8000000000000002,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "stages.champion.watch_power": {
          "mean": 0.5,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "stages.champion.cross_strength": {
          "mean": 0.5,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "paper_results.offline_ndcg50_gain_percent": {
          "mean": 19.9,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "paper_results.online_user_satisfaction_percent": {
          "mean": 3.77,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "paper_results.online_unique_content_percent": {
          "mean": 7.44,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        },
        "paper_results.autonomous_runs": {
          "mean": 41.0,
          "std": 0.0,
          "ci95": 0.0,
          "n": 3
        }
      },
      "formal_comparison": true
    }
  },
  "manifest_ref": "reproduction:recevolve",
  "evaluation_protocol": {
    "tier": "l2_public_dataset",
    "seeds": [
      42,
      43,
      44
    ],
    "formal_comparison": true,
    "claim_policy": "formal multi-seed comparison"
  },
  "provenance": {
    "artifact_path": "docs/reproductions/2609.01622-recevolve/metrics/public-seeds42-44.json",
    "dataset_fingerprint": "not recorded in historical artifact"
  }
}