{
  "rows": 288,
  "unique_rows": 288,
  "successful_raw_responses": 144,
  "model": "jev-1.13.0",
  "input_tokens": 850704,
  "estimated_usd": 0.035729567999999996,
  "frozen_hashes_valid": true,
  "remote_rankings_reconstructed": true,
  "summary_recomputed": true,
  "shortlist_exclusions": [
    {
      "scale": 96,
      "query": "Q07",
      "missing": [
        "D006"
      ]
    },
    {
      "scale": 96,
      "query": "Q19",
      "missing": [
        "D010"
      ]
    },
    {
      "scale": 240,
      "query": "Q07",
      "missing": [
        "D006"
      ]
    },
    {
      "scale": 240,
      "query": "Q19",
      "missing": [
        "D010"
      ]
    }
  ],
  "summary": [
    {
      "scale": 24,
      "method": "bm25",
      "hit1_count": 15,
      "hit1": 0.75,
      "recall3": 0.85,
      "all_evidence3": 0.85,
      "negative_specificity": 0.0,
      "decision_accuracy": 0.625,
      "p50_ms": 0.674800481647253,
      "input_tokens": 0
    },
    {
      "scale": 24,
      "method": "embedding",
      "hit1_count": 17,
      "hit1": 0.85,
      "recall3": 1.0,
      "all_evidence3": 1.0,
      "negative_specificity": 0.5,
      "decision_accuracy": 0.625,
      "p50_ms": 64.41805511713028,
      "input_tokens": 0
    },
    {
      "scale": 24,
      "method": "hybrid",
      "hit1_count": 20,
      "hit1": 1.0,
      "recall3": 1.0,
      "all_evidence3": 1.0,
      "negative_specificity": 1.0,
      "decision_accuracy": 1.0,
      "p50_ms": 174.71241287421435,
      "input_tokens": 26350
    },
    {
      "scale": 24,
      "method": "direct",
      "hit1_count": 20,
      "hit1": 1.0,
      "recall3": 1.0,
      "all_evidence3": 1.0,
      "negative_specificity": 1.0,
      "decision_accuracy": 1.0,
      "p50_ms": 172.4091898649931,
      "input_tokens": 52685
    },
    {
      "scale": 96,
      "method": "bm25",
      "hit1_count": 16,
      "hit1": 0.8,
      "recall3": 0.825,
      "all_evidence3": 0.8,
      "negative_specificity": 0.0,
      "decision_accuracy": 0.6666666666666666,
      "p50_ms": 1.1050445027649403,
      "input_tokens": 0
    },
    {
      "scale": 96,
      "method": "embedding",
      "hit1_count": 17,
      "hit1": 0.85,
      "recall3": 1.0,
      "all_evidence3": 1.0,
      "negative_specificity": 0.5,
      "decision_accuracy": 0.625,
      "p50_ms": 63.43889550771564,
      "input_tokens": 0
    },
    {
      "scale": 96,
      "method": "hybrid",
      "hit1_count": 19,
      "hit1": 0.95,
      "recall3": 0.925,
      "all_evidence3": 0.9,
      "negative_specificity": 1.0,
      "decision_accuracy": 0.9583333333333334,
      "p50_ms": 163.83259952999651,
      "input_tokens": 27188
    },
    {
      "scale": 96,
      "method": "direct",
      "hit1_count": 20,
      "hit1": 1.0,
      "recall3": 1.0,
      "all_evidence3": 1.0,
      "negative_specificity": 1.0,
      "decision_accuracy": 1.0,
      "p50_ms": 202.0702165318653,
      "input_tokens": 206573
    },
    {
      "scale": 240,
      "method": "bm25",
      "hit1_count": 16,
      "hit1": 0.8,
      "recall3": 0.825,
      "all_evidence3": 0.8,
      "negative_specificity": 0.0,
      "decision_accuracy": 0.6666666666666666,
      "p50_ms": 1.8616124289110303,
      "input_tokens": 0
    },
    {
      "scale": 240,
      "method": "embedding",
      "hit1_count": 17,
      "hit1": 0.85,
      "recall3": 1.0,
      "all_evidence3": 1.0,
      "negative_specificity": 0.5,
      "decision_accuracy": 0.625,
      "p50_ms": 61.26406288240105,
      "input_tokens": 0
    },
    {
      "scale": 240,
      "method": "hybrid",
      "hit1_count": 19,
      "hit1": 0.95,
      "recall3": 0.925,
      "all_evidence3": 0.9,
      "negative_specificity": 1.0,
      "decision_accuracy": 0.9583333333333334,
      "p50_ms": 161.59235849045217,
      "input_tokens": 27015
    },
    {
      "scale": 240,
      "method": "direct",
      "hit1_count": 20,
      "hit1": 1.0,
      "recall3": 1.0,
      "all_evidence3": 1.0,
      "negative_specificity": 1.0,
      "decision_accuracy": 1.0,
      "p50_ms": 297.40516550373286,
      "input_tokens": 510893
    }
  ],
  "source_sha256": {
    "results.jsonl": "39bc8035ebf73f3c8c377a305dee120c87bf72cd73f31f9d4d21ebf1949e7e73",
    "corpus.json": "df3c9df1897e07b4b833dbb5762e2f821413d070a2ce4480dfe860699f6960ec",
    "queries.json": "4c883138e609b103eabdb60ab7c948732a5c28357fa3472050d9ca2e5ea83d6d",
    "summary.json": "ae23577f519fc1a274c8862a95710e8778a5bcf0dd28a27509c4eae8e9964b61",
    "run_benchmark.py": "ed28000b68ee61b3be70df1b9c1090639f27ed0479bc97f5d26fdb4613be3d61",
    "analyze.py": "d85e769b94898ed34e15ca16c09a10d6dd59c822386a9b3269675d0033197afc"
  }
}