{
  "oags": "0.2",
  "status": "alpha",
  "entry": {
    "id": "IB-arxiv-2607.28367-broken-tasks",
    "type": "evidence",
    "claim": "In an audit of 150 FAIL-scored computer-use-agent trajectories, 4.7% of FAIL verdicts stem from broken tasks.",
    "evidence_grade": "E3",
    "state": "active",
    "primary_source_ref": [
      {
        "kind": "arxiv",
        "id": "2607.28367",
        "version": "v1",
        "url": "https://arxiv.org/abs/2607.28367",
        "locator": "abstract"
      }
    ],
    "quantities": {
      "broken_task_pct": 4.7,
      "n_trajectories": 150
    },
    "declared_loss": null,
    "source_title": "How Benchmarks Mis-Score Computer-Use Agents",
    "authors": [
      "Zihan Dong",
      "Zhiyuan Ma",
      "Zekun Wang",
      "Yunqing Li",
      "Zirou Liu",
      "Ruixuan Deng",
      "Qishi Zhan",
      "Rui Qian"
    ],
    "result_origin": "measured",
    "support_quote": "audit 150 public failure-scored trajectories from five web, enterprise-workflow, and desktop-control benchmarks, find that 15.3\\% of FAIL verdicts are wrong: 10.7\\% are evaluator false negatives and 4.7\\% are broken tasks",
    "source_version": "v1",
    "abstract_check": {
      "version": "v1",
      "date": "2026-10-08",
      "quote_verbatim": true
    },
    "domain_v2": "cs-cl/llm-evaluation"
  },
  "expand": [],
  "incoming": [],
  "links": [],
  "declared_losses": [
    "Public evidence-graph projection: internal analytic/theory edges and private provenance are withheld by design.",
    "Alpha: some primary-source refs await precise citation verification."
  ],
  "generated": "2026-10-08"
}