{
  "annotation_id": "722ca020-eb88-4461-bc61-2aa31c0bc366",
  "date_created": "2026-02-17T05:53:15.373000Z",
  "date_modified": "2026-02-17T05:53:15.373000Z",
  "file_hash": "d25afbb77a03115d1c6bb9d1b8fb26097b33a896985f158d0539307d816f075b",
  "private": false,
  "record": {
    "abstract": "The miscalibration of Large Reasoning Models (LRMs) undermines their reliability in high-stakes domains, necessitating methods to accurately estimate the confidence of their long-form, multi-step outputs. To address this gap, we introduce the Reasoning Model Confidence estimation Benchmark (RMCB), a public resource of 347,496 reasoning traces from six popular LRMs across different architectural families. The benchmark is constructed from a diverse suite of datasets spanning high-stakes domains, including clinical, financial, legal, and mathematical reasoning, alongside complex general reasoning benchmarks, with correctness annotations provided for all samples. Using RMCB, we conduct a large-scale empirical evaluation of over ten distinct representation-based methods, spanning sequential, graph-based, and text-based architectures. Our central finding is a persistent trade-off between discrimination (AUROC) and calibration (ECE): text-based encoders achieve the best AUROC (0.672), while structurally-aware models yield the best ECE (0.148), with no single method dominating both. Furthermore, we find that increased architectural complexity does not reliably outperform simpler sequential baselines, suggesting a performance ceiling for methods relying solely on chunk-level hidden states. This work provides the most comprehensive benchmark for this task to date, establishing rigorous baselines and demonstrating the limitations of current representation-based paradigms.",
    "arxiv_id": "2601.08134",
    "authors": [
      "Reza Khanmohammadi",
      "Erfan Miahi",
      "Simerjot Kaur",
      "Ivan Brugere",
      "Charese H. Smiley",
      "Kundan Thind",
      "Mohammad M. Ghassemi"
    ],
    "categories": [
      "cs.CL"
    ],
    "license": "http://creativecommons.org/licenses/by/4.0/",
    "title": "How Reliable are Confidence Estimators for Large Reasoning Models? A Systematic Benchmark on High-Stakes Domains",
    "url": "https://arxiv.org/abs/2601.08134",
    "version": "v1"
  },
  "schema_id": "dorsal/arxiv",
  "source": {
    "execution_id": "914186ee-0c58-48db-8374-a07bfa8ea12a",
    "id": "arXiv Dataset",
    "type": "Model",
    "variant": "snapshot-2026-01-17",
    "version": "0.1.0"
  },
  "user_id": 1000002
}