{
  "metadata": {
    "version": 2,
    "title": "AutoRSE leaderboard reference results",
    "source": "AutoRSE, Table 1: Final performance across six recommendation tasks",
    "paper_url": "https://arxiv.org/abs/2610.37228",
    "code_url": "https://github.com/Ray-Tiann/AutoRSE",
    "provenance": "Paper reference",
    "prepared_date": "2026-10-30",
    "configuration_count": 7,
    "task_count": 6,
    "performance_cell_count": 42,
    "run_budget_hours": 12,
    "score_definition": "Mean final held-out test score; sd is the reported standard deviation across available performance trials.",
    "relative_gain_definition": "100 × (mean final test score / task baseline score − 1), reported to two decimal places.",
    "overall_ranking_metric": "mean_task_rank",
    "overall_ranking_direction": "ascending",
    "overall_ranking_definition": "Equal-weight mean of the six task ranks; lower is better. Rank sums and means produce identical overall ordering for the same six tasks.",
    "task_ranking_definition": "Within the complete six-task cohort, higher reported mean test scores rank better. Scores are compared at their published four-decimal precision.",
    "task_tie_policy": "Midranks: tied task scores share the average of their occupied positions, for example 1, 2.5, 2.5, 4.",
    "overall_tie_policy": "Equal exact rank sums share a competition rank, for example 1, 1, 3. Rounded display means do not determine ties.",
    "overall_cohort_definition": "Only entries with scores and gains for all six tasks enter the overall cohort. task_ranks, rank_sum, mean_task_rank and overall_rank snapshot this fixed cohort; incomplete entries have null derived fields.",
    "overall_cohort_count": 7,
    "filter_policy": "Harness filters only hide rows and do not recalculate task ranks, mean task ranks, or overall ranks.",
    "partial_results_policy": "Incomplete submissions are excluded from the overall cohort. A per-task view can rank all entries with an available score on that task separately.",
    "average_gain_definition": "Published unweighted mean relative test-score change across all six tasks, including negative gains. Retained as a reported statistic, not the overall ranking metric. Raw AUC and NDCG scores are not averaged together.",
    "precision": "Published scores and SDs (four decimals) and published per-task and average gains (two decimals) are preserved. Average gains are not recomputed from the rounded per-task percentages.",
    "trial_count_definition": "Three available performance trials per task–configuration, except OneTrans with Terminus-2 × GPT-5.5, which has two. trial_counts follows task_order.",
    "ranking_caution": "Descriptive ordering of reported means; no claim of statistically significant differences between adjacent entries.",
    "source_sha256": {
      "website_data_js": "39d912addc597f9a9e24d4edf07a2637ae76920d8017550bb5a4c8508eac2348",
      "novelty_json": "a8a48730c63f792d5b88fad99b790ad06e646cb8b3a6b1e2391148d5d94db471",
      "manuscript_experiments_tex": "f31a9252198beed6f950850595aa6afd3ace1e3fb309355fc34724db9e052493"
    }
  },
  "task_order": [
    "HSTU",
    "IntSR",
    "TIGER",
    "OneTrans",
    "RankMixer",
    "WuKong"
  ],
  "tasks": [
    {
      "name": "HSTU",
      "type": "Generative recommendation",
      "dataset": "Amazon Beauty",
      "metric": "NDCG@10",
      "baseline_score": 0.0535,
      "method": "Sequential modeling"
    },
    {
      "name": "IntSR",
      "type": "Generative recommendation",
      "dataset": "Amazon Kindle",
      "metric": "Macro NDCG@10",
      "baseline_score": 0.694,
      "method": "Joint search–recommendation modeling"
    },
    {
      "name": "TIGER",
      "type": "Generative recommendation",
      "dataset": "Amazon Beauty",
      "metric": "NDCG@10",
      "baseline_score": 0.0332,
      "method": "Semantic-ID generation"
    },
    {
      "name": "OneTrans",
      "type": "CTR prediction",
      "dataset": "KuaiSAR v2",
      "metric": "Click AUC",
      "baseline_score": 0.6767,
      "method": "Unified sequence modeling"
    },
    {
      "name": "RankMixer",
      "type": "CTR prediction",
      "dataset": "Criteo 5M",
      "metric": "AUC",
      "baseline_score": 0.7859,
      "method": "Multi-head token mixing"
    },
    {
      "name": "WuKong",
      "type": "CTR prediction",
      "dataset": "MovieLens Latest",
      "metric": "AUC",
      "baseline_score": 0.9645,
      "method": "Feature interaction"
    }
  ],
  "rows": [
    {
      "harness": "Claude Code",
      "model": "DeepSeek V4 Flash",
      "scores": [
        0.0674,
        0.6606,
        0.0476,
        0.7467,
        0.7979,
        0.9805
      ],
      "sd": [
        0.0032,
        0.0541,
        0.0138,
        0.0324,
        0.0015,
        0.003
      ],
      "relative_gain_percent": [
        25.98,
        -4.81,
        43.37,
        10.34,
        1.53,
        1.66
      ],
      "average_gain_percent": 13.01,
      "id": "claude-code-deepseek-v4-flash",
      "provenance": "Paper reference",
      "source_url": "https://arxiv.org/abs/2610.37228",
      "trial_counts": [
        3,
        3,
        3,
        3,
        3,
        3
      ],
      "task_ranks": [
        3,
        4,
        6,
        1,
        3,
        4
      ],
      "rank_sum": 21,
      "mean_task_rank": 3.5,
      "overall_rank": 3
    },
    {
      "harness": "Claude Code",
      "model": "Claude Opus 4.8",
      "scores": [
        0.0754,
        0.7325,
        0.0631,
        0.7184,
        0.8002,
        0.984
      ],
      "sd": [
        0.0043,
        0.0292,
        0.0164,
        0.0068,
        0.0009,
        0.0039
      ],
      "relative_gain_percent": [
        40.93,
        5.55,
        90.06,
        6.16,
        1.82,
        2.02
      ],
      "average_gain_percent": 24.42,
      "id": "claude-code-claude-opus-4-8",
      "provenance": "Paper reference",
      "source_url": "https://arxiv.org/abs/2610.37228",
      "trial_counts": [
        3,
        3,
        3,
        3,
        3,
        3
      ],
      "task_ranks": [
        1,
        2,
        2,
        6,
        1,
        2
      ],
      "rank_sum": 14,
      "mean_task_rank": 2.3333333333333335,
      "overall_rank": 1
    },
    {
      "harness": "Codex",
      "model": "DeepSeek V4 Flash",
      "scores": [
        0.0657,
        0.6485,
        0.0583,
        0.7203,
        0.7974,
        0.979
      ],
      "sd": [
        0.003,
        0.0335,
        0.0147,
        0.0044,
        0.0003,
        0.0018
      ],
      "relative_gain_percent": [
        22.8,
        -6.56,
        75.6,
        6.44,
        1.46,
        1.5
      ],
      "average_gain_percent": 16.88,
      "id": "codex-deepseek-v4-flash",
      "provenance": "Paper reference",
      "source_url": "https://arxiv.org/abs/2610.37228",
      "trial_counts": [
        3,
        3,
        3,
        3,
        3,
        3
      ],
      "task_ranks": [
        6,
        5,
        4,
        5,
        5,
        6
      ],
      "rank_sum": 31,
      "mean_task_rank": 5.166666666666667,
      "overall_rank": 7
    },
    {
      "harness": "Codex",
      "model": "GPT-5.5",
      "scores": [
        0.0672,
        0.7692,
        0.0659,
        0.7292,
        0.7954,
        0.9796
      ],
      "sd": [
        0.0065,
        0.1904,
        0.0046,
        0.0342,
        0.003,
        0.0032
      ],
      "relative_gain_percent": [
        25.61,
        10.84,
        98.49,
        7.76,
        1.21,
        1.57
      ],
      "average_gain_percent": 24.24,
      "id": "codex-gpt-5-5",
      "provenance": "Paper reference",
      "source_url": "https://arxiv.org/abs/2610.37228",
      "trial_counts": [
        3,
        3,
        3,
        3,
        3,
        3
      ],
      "task_ranks": [
        4,
        1,
        1,
        2,
        6,
        5
      ],
      "rank_sum": 19,
      "mean_task_rank": 3.1666666666666665,
      "overall_rank": 2
    },
    {
      "harness": "Terminus-2",
      "model": "DeepSeek V4 Flash",
      "scores": [
        0.0667,
        0.6291,
        0.0554,
        0.6665,
        0.7977,
        0.9877
      ],
      "sd": [
        0.0057,
        0.0858,
        0.0158,
        0.0447,
        0.0012,
        0.002
      ],
      "relative_gain_percent": [
        24.67,
        -9.35,
        66.87,
        -1.51,
        1.5,
        2.41
      ],
      "average_gain_percent": 14.1,
      "id": "terminus-2-deepseek-v4-flash",
      "provenance": "Paper reference",
      "source_url": "https://arxiv.org/abs/2610.37228",
      "trial_counts": [
        3,
        3,
        3,
        3,
        3,
        3
      ],
      "task_ranks": [
        5,
        6,
        5,
        7,
        4,
        1
      ],
      "rank_sum": 28,
      "mean_task_rank": 4.666666666666667,
      "overall_rank": 5
    },
    {
      "harness": "Terminus-2",
      "model": "GPT-5.5",
      "scores": [
        0.0697,
        0.6146,
        0.0614,
        0.7285,
        0.7944,
        0.9749
      ],
      "sd": [
        0.0058,
        0.0711,
        0.0053,
        0.0498,
        0.0021,
        0.0035
      ],
      "relative_gain_percent": [
        30.28,
        -11.44,
        84.94,
        7.65,
        1.08,
        1.08
      ],
      "average_gain_percent": 18.93,
      "id": "terminus-2-gpt-5-5",
      "provenance": "Paper reference",
      "source_url": "https://arxiv.org/abs/2610.37228",
      "trial_counts": [
        3,
        3,
        3,
        2,
        3,
        3
      ],
      "task_ranks": [
        2,
        7,
        3,
        3,
        7,
        7
      ],
      "rank_sum": 29,
      "mean_task_rank": 4.833333333333333,
      "overall_rank": 6
    },
    {
      "harness": "Terminus-2",
      "model": "Claude Opus 4.8",
      "scores": [
        0.0636,
        0.6761,
        0.0273,
        0.7207,
        0.7989,
        0.9823
      ],
      "sd": [
        0.0015,
        0.0334,
        0.019,
        0.0232,
        0.0014,
        0.0004
      ],
      "relative_gain_percent": [
        18.88,
        -2.58,
        -17.77,
        6.5,
        1.65,
        1.85
      ],
      "average_gain_percent": 1.42,
      "id": "terminus-2-claude-opus-4-8",
      "provenance": "Paper reference",
      "source_url": "https://arxiv.org/abs/2610.37228",
      "trial_counts": [
        3,
        3,
        3,
        3,
        3,
        3
      ],
      "task_ranks": [
        7,
        3,
        7,
        4,
        2,
        3
      ],
      "rank_sum": 26,
      "mean_task_rank": 4.333333333333333,
      "overall_rank": 4
    }
  ]
}