{
  "schemaVersion": 2,
  "source": {
    "paperRevision": "a128f917d440d914a3004dc9e1ddb55386e899ee",
    "note": "Benchmark display values transcribed from the current manuscript tables.",
    "overviewScope": "Benchmark-focused excerpt of the manuscript abstract.",
    "metricProtocolVersion": "5.5",
    "metricProtocolSha256": "8ffa7d5ceae792f63e832434f6b20b7b70148391962bd79011583bcc3ecf2b01",
    "files": {
      "arxiv/abstract.tex": "52d88ac1fe14f39a655f9d040fec2a6e08587947ec2e51382c6bf2b2245a20cd",
      "arxiv/table/tab_main_results_dev15.tex": "826955d96a28589871595a406ab5dd4e884c1cf9da74b55d760b4487416a593e",
      "arxiv/data/main_results.json": "a43257b1417d3b29210a5674d7cdde368bde388817b9320a322fc9828e62fd60",
      "arxiv/table/tab_action_duration.tex": "24d970a0b3e42df89368c93b6596e219408dbc13f0e60d1e76dd05ae0b50fa4e",
      "arxiv/data/action_duration/heatmap_scores.csv": "befe981b25e2a1eba5f9e42fd7f5144f9b1ba58802188b0d2004f4d752a10965",
      "arxiv/data/action_duration/provenance_v55.json": "cf60342c3473cb2afa52a656b6ff08fd30bbc2ce41255689b82f789ffbd816b8"
    }
  },
  "overview": "Learning from human data offers a scalable path toward embodied intelligence, yet it requires models to accurately understand fine-grained human actions from egocentric observations. However, existing benchmarks remain limited in action atomicity, temporal precision, and hand–action association. We introduce EgoActionBench, a benchmark that represents continuous human manipulation as temporally localized atomic actions associated with the left hand, right hand, or both hands. EgoActionBench contains 1,407 episodes with 41,132 human-refined action annotations. To evaluate such dense predictions, we propose a progressive evaluation protocol that retrieves temporally eligible candidates, selects a prediction by content agreement, and scores semantic fidelity and temporal overlap alongside action precision and recall, jointly capturing what happens, when it happens, and which hand performs it. We benchmark 19 vision-language models on EgoActionBench. Our experiments reveal substantial limitations of current models in action atomicity, temporal precision, and hand–action association, highlighting the gap between general video understanding and the action understanding required for embodied intelligence.",
  "overall": [
    {
      "model": "Human Expert",
      "displayName": "Human Expert",
      "group": "Human Baseline",
      "ours": false,
      "reference": true,
      "scores": {
        "EgoActionScore": 90.4,
        "m_tIoU": 88.4,
        "F1": 93.1,
        "Temporal": 90.8,
        "Hand": 95.5,
        "Verb": 94.5,
        "Objects": 95.1,
        "Tools": 94.8,
        "Motion": 74.5,
        "Semantic": 90.1
      }
    },
    {
      "model": "GPT-5.6 Sol",
      "displayName": "GPT-5.6 Sol",
      "group": "Proprietary models",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 49.8,
        "m_tIoU": 31.0,
        "F1": 57.6,
        "Temporal": 44.3,
        "Hand": 67.1,
        "Verb": 50.8,
        "Objects": 60.5,
        "Tools": 77.4,
        "Motion": 41.5,
        "Semantic": 55.2
      }
    },
    {
      "model": "GPT-6 Astra",
      "displayName": "GPT-6 Astra",
      "group": "Proprietary models",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 59.6,
        "m_tIoU": 40.6,
        "F1": 68.6,
        "Temporal": 54.6,
        "Hand": 75.8,
        "Verb": 63.3,
        "Objects": 69.9,
        "Tools": 80.8,
        "Motion": 48.9,
        "Semantic": 64.6
      }
    },
    {
      "model": "Gemini 3.1 Pro Preview",
      "displayName": "Gemini 3.1 Pro Preview",
      "group": "Proprietary models",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 34.8,
        "m_tIoU": 18.1,
        "F1": 41.8,
        "Temporal": 29.9,
        "Hand": 49.4,
        "Verb": 34.5,
        "Objects": 43.6,
        "Tools": 64.5,
        "Motion": 30.7,
        "Semantic": 39.8
      }
    },
    {
      "model": "Qwen2.5-VL-72B-Instruct",
      "displayName": "Qwen2.5-VL-72B",
      "group": "Open-source VLMs",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 24.7,
        "m_tIoU": 8.2,
        "F1": 26.1,
        "Temporal": 17.2,
        "Hand": 43.4,
        "Verb": 26.8,
        "Objects": 32.8,
        "Tools": 49.3,
        "Motion": 25.4,
        "Semantic": 32.2
      }
    },
    {
      "model": "Qwen3-VL-32B-Instruct",
      "displayName": "Qwen3-VL-32B",
      "group": "Open-source VLMs",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 29.2,
        "m_tIoU": 11.8,
        "F1": 32.0,
        "Temporal": 21.9,
        "Hand": 48.5,
        "Verb": 30.8,
        "Objects": 37.2,
        "Tools": 58.2,
        "Motion": 29.2,
        "Semantic": 36.6
      }
    },
    {
      "model": "Qwen3-VL-8B-Instruct",
      "displayName": "Qwen3-VL-8B",
      "group": "Open-source VLMs",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 25.2,
        "m_tIoU": 10.3,
        "F1": 28.1,
        "Temporal": 19.2,
        "Hand": 43.7,
        "Verb": 25.5,
        "Objects": 32.2,
        "Tools": 51.5,
        "Motion": 23.0,
        "Semantic": 31.2
      }
    },
    {
      "model": "InternVL3.5-8B",
      "displayName": "InternVL3.5-8B",
      "group": "Open-source VLMs",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 20.4,
        "m_tIoU": 8.8,
        "F1": 25.4,
        "Temporal": 17.1,
        "Hand": 32.4,
        "Verb": 19.4,
        "Objects": 25.5,
        "Tools": 44.6,
        "Motion": 16.9,
        "Semantic": 23.7
      }
    },
    {
      "model": "MiniCPM-V-4.5-8B",
      "displayName": "MiniCPM-V-4.5-8B",
      "group": "Open-source VLMs",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 19.4,
        "m_tIoU": 6.7,
        "F1": 20.3,
        "Temporal": 13.5,
        "Hand": 33.9,
        "Verb": 21.9,
        "Objects": 25.2,
        "Tools": 34.5,
        "Motion": 20.3,
        "Semantic": 25.3
      }
    },
    {
      "model": "Molmo2-8B",
      "displayName": "Molmo2-8B",
      "group": "Open-source VLMs",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 18.8,
        "m_tIoU": 8.0,
        "F1": 21.2,
        "Temporal": 14.6,
        "Hand": 30.4,
        "Verb": 19.1,
        "Objects": 24.6,
        "Tools": 41.3,
        "Motion": 17.3,
        "Semantic": 23.0
      }
    },
    {
      "model": "LLaVA-OV2-8B",
      "displayName": "LLaVA-OV2-8B",
      "group": "Open-source VLMs",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 24.7,
        "m_tIoU": 9.8,
        "F1": 26.7,
        "Temporal": 18.2,
        "Hand": 42.0,
        "Verb": 25.8,
        "Objects": 32.6,
        "Tools": 48.1,
        "Motion": 23.9,
        "Semantic": 31.2
      }
    },
    {
      "model": "VideoLLaMA3-7B",
      "displayName": "VideoLLaMA3-7B",
      "group": "Open-source VLMs",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 14.2,
        "m_tIoU": 4.4,
        "F1": 11.1,
        "Temporal": 7.8,
        "Hand": 24.1,
        "Verb": 17.4,
        "Objects": 24.8,
        "Tools": 38.6,
        "Motion": 16.3,
        "Semantic": 20.7
      }
    },
    {
      "model": "Qwen3-VL-4B-Instruct",
      "displayName": "Qwen3-VL-4B",
      "group": "Open-source VLMs",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 25.9,
        "m_tIoU": 9.9,
        "F1": 28.2,
        "Temporal": 19.1,
        "Hand": 46.3,
        "Verb": 25.2,
        "Objects": 35.0,
        "Tools": 47.8,
        "Motion": 24.0,
        "Semantic": 32.8
      }
    },
    {
      "model": "CapRL-Video-4B",
      "displayName": "CapRL-Video-4B",
      "group": "Open-source video captioning models",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 32.4,
        "m_tIoU": 16.9,
        "F1": 40.1,
        "Temporal": 28.5,
        "Hand": 49.3,
        "Verb": 28.7,
        "Objects": 39.0,
        "Tools": 60.6,
        "Motion": 27.5,
        "Semantic": 36.3
      }
    },
    {
      "model": "OmniCaptioner-IF-3B",
      "displayName": "OmniCaptioner-IF-3B",
      "group": "Open-source video captioning models",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 20.0,
        "m_tIoU": 6.6,
        "F1": 25.7,
        "Temporal": 16.1,
        "Hand": 36.0,
        "Verb": 18.0,
        "Objects": 21.9,
        "Tools": 28.8,
        "Motion": 19.5,
        "Semantic": 23.8
      }
    },
    {
      "model": "OmniCaptioner-IF-7B",
      "displayName": "OmniCaptioner-IF-7B",
      "group": "Open-source video captioning models",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 23.0,
        "m_tIoU": 10.1,
        "F1": 25.0,
        "Temporal": 17.6,
        "Hand": 38.5,
        "Verb": 24.3,
        "Objects": 26.3,
        "Tools": 40.8,
        "Motion": 24.6,
        "Semantic": 28.5
      }
    },
    {
      "model": "video-SALMONN-2+-3B",
      "displayName": "video-SALMONN-2+-3B",
      "group": "Open-source video captioning models",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 10.8,
        "m_tIoU": 3.5,
        "F1": 14.2,
        "Temporal": 8.8,
        "Hand": 19.9,
        "Verb": 8.5,
        "Objects": 13.7,
        "Tools": 28.4,
        "Motion": 9.0,
        "Semantic": 12.8
      }
    },
    {
      "model": "EgoThinker-7B",
      "displayName": "EgoThinker-7B",
      "group": "Open-source egocentric video understanding models",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 20.8,
        "m_tIoU": 13.7,
        "F1": 24.1,
        "Temporal": 18.9,
        "Hand": 33.7,
        "Verb": 19.0,
        "Objects": 19.1,
        "Tools": 13.1,
        "Motion": 19.5,
        "Semantic": 22.8
      }
    },
    {
      "model": "EgoGPT-7B",
      "displayName": "EgoGPT-7B",
      "group": "Open-source egocentric video understanding models",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 15.4,
        "m_tIoU": 4.5,
        "F1": 16.0,
        "Temporal": 10.3,
        "Hand": 24.1,
        "Verb": 18.9,
        "Objects": 18.4,
        "Tools": 22.5,
        "Motion": 20.3,
        "Semantic": 20.4
      }
    },
    {
      "model": "RynnBrain-4B",
      "displayName": "RynnBrain-4B",
      "group": "Open-source egocentric video understanding models",
      "ours": false,
      "reference": false,
      "scores": {
        "EgoActionScore": 13.9,
        "m_tIoU": 10.4,
        "F1": 13.3,
        "Temporal": 11.8,
        "Hand": 21.2,
        "Verb": 12.9,
        "Objects": 15.4,
        "Tools": 15.7,
        "Motion": 14.4,
        "Semantic": 15.9
      }
    },
    {
      "model": "EgoAction (LLaVA-OV2-8B)",
      "displayName": "EgoAction (LLaVA-OV2-8B)",
      "group": "EgoAction (Ours)",
      "ours": true,
      "reference": false,
      "scores": {
        "EgoActionScore": 47.9,
        "m_tIoU": 32.3,
        "F1": 57.0,
        "Temporal": 44.6,
        "Hand": 67.0,
        "Verb": 46.2,
        "Objects": 51.7,
        "Tools": 63.0,
        "Motion": 39.6,
        "Semantic": 51.2
      }
    },
    {
      "model": "EgoAction (Qwen3-VL-4B)",
      "displayName": "EgoAction (Qwen3-VL-4B)",
      "group": "EgoAction (Ours)",
      "ours": true,
      "reference": false,
      "scores": {
        "EgoActionScore": 41.8,
        "m_tIoU": 26.6,
        "F1": 45.9,
        "Temporal": 36.3,
        "Hand": 63.4,
        "Verb": 41.5,
        "Objects": 47.6,
        "Tools": 58.2,
        "Motion": 36.8,
        "Semantic": 47.4
      }
    }
  ],
  "duration": [
    {
      "model": "Qwen3-VL-4B",
      "displayName": "Qwen3-VL-4B",
      "ours": false,
      "metric": "Content",
      "values": [
        25.3,
        26.6,
        30.4,
        34.9,
        42.2
      ]
    },
    {
      "model": "Qwen3-VL-4B",
      "displayName": "Qwen3-VL-4B",
      "ours": false,
      "metric": "Semantic",
      "values": [
        29.5,
        31.3,
        34.9,
        39.6,
        46.6
      ]
    },
    {
      "model": "Qwen3-VL-4B",
      "displayName": "Qwen3-VL-4B",
      "ours": false,
      "metric": "m_tIoU",
      "values": [
        3.6,
        7.9,
        13.4,
        21.3,
        39.1
      ]
    },
    {
      "model": "Qwen3-VL-4B",
      "displayName": "Qwen3-VL-4B",
      "ours": false,
      "metric": "Hand",
      "values": [
        42.3,
        45.4,
        48.5,
        54.2,
        60.8
      ]
    },
    {
      "model": "EgoAction (Qwen3-VL-4B)",
      "displayName": "EgoAction (Qwen3-VL-4B)",
      "ours": true,
      "metric": "Content",
      "values": [
        35.0,
        41.8,
        48.0,
        53.5,
        59.4
      ]
    },
    {
      "model": "EgoAction (Qwen3-VL-4B)",
      "displayName": "EgoAction (Qwen3-VL-4B)",
      "ours": true,
      "metric": "Semantic",
      "values": [
        39.8,
        47.1,
        53.5,
        59.2,
        65.1
      ]
    },
    {
      "model": "EgoAction (Qwen3-VL-4B)",
      "displayName": "EgoAction (Qwen3-VL-4B)",
      "ours": true,
      "metric": "m_tIoU",
      "values": [
        18.4,
        26.1,
        30.3,
        40.4,
        56.1
      ]
    },
    {
      "model": "EgoAction (Qwen3-VL-4B)",
      "displayName": "EgoAction (Qwen3-VL-4B)",
      "ours": true,
      "metric": "Hand",
      "values": [
        54.4,
        63.2,
        70.5,
        77.4,
        83.4
      ]
    },
    {
      "model": "LLaVA-OV2-8B",
      "displayName": "LLaVA-OV2-8B",
      "ours": false,
      "metric": "Content",
      "values": [
        23.3,
        25.9,
        30.8,
        36.6,
        43.9
      ]
    },
    {
      "model": "LLaVA-OV2-8B",
      "displayName": "LLaVA-OV2-8B",
      "ours": false,
      "metric": "Semantic",
      "values": [
        26.5,
        29.4,
        34.6,
        40.6,
        48.4
      ]
    },
    {
      "model": "LLaVA-OV2-8B",
      "displayName": "LLaVA-OV2-8B",
      "ours": false,
      "metric": "m_tIoU",
      "values": [
        3.2,
        7.6,
        13.6,
        21.8,
        40.5
      ]
    },
    {
      "model": "LLaVA-OV2-8B",
      "displayName": "LLaVA-OV2-8B",
      "ours": false,
      "metric": "Hand",
      "values": [
        36.1,
        40.2,
        46.3,
        53.3,
        63.4
      ]
    },
    {
      "model": "EgoAction (LLaVA-OV2-8B)",
      "displayName": "EgoAction (LLaVA-OV2-8B)",
      "ours": true,
      "metric": "Content",
      "values": [
        39.3,
        45.8,
        52.0,
        56.1,
        61.5
      ]
    },
    {
      "model": "EgoAction (LLaVA-OV2-8B)",
      "displayName": "EgoAction (LLaVA-OV2-8B)",
      "ours": true,
      "metric": "Semantic",
      "values": [
        44.1,
        51.1,
        57.4,
        61.4,
        66.5
      ]
    },
    {
      "model": "EgoAction (LLaVA-OV2-8B)",
      "displayName": "EgoAction (LLaVA-OV2-8B)",
      "ours": true,
      "metric": "m_tIoU",
      "values": [
        20.3,
        33.5,
        40.3,
        48.9,
        59.3
      ]
    },
    {
      "model": "EgoAction (LLaVA-OV2-8B)",
      "displayName": "EgoAction (LLaVA-OV2-8B)",
      "ours": true,
      "metric": "Hand",
      "values": [
        58.8,
        67.2,
        73.9,
        78.2,
        83.0
      ]
    },
    {
      "model": "Qwen3-VL-32B",
      "displayName": "Qwen3-VL-32B",
      "ours": false,
      "metric": "Content",
      "values": [
        28.1,
        30.5,
        36.7,
        41.8,
        50.0
      ]
    },
    {
      "model": "Qwen3-VL-32B",
      "displayName": "Qwen3-VL-32B",
      "ours": false,
      "metric": "Semantic",
      "values": [
        31.8,
        34.6,
        40.5,
        45.9,
        54.5
      ]
    },
    {
      "model": "Qwen3-VL-32B",
      "displayName": "Qwen3-VL-32B",
      "ours": false,
      "metric": "m_tIoU",
      "values": [
        4.3,
        9.6,
        15.9,
        25.0,
        45.3
      ]
    },
    {
      "model": "Qwen3-VL-32B",
      "displayName": "Qwen3-VL-32B",
      "ours": false,
      "metric": "Hand",
      "values": [
        42.9,
        47.0,
        52.4,
        58.7,
        69.2
      ]
    },
    {
      "model": "GPT-6 Astra",
      "displayName": "GPT-6 Astra",
      "ours": false,
      "metric": "Content",
      "values": [
        51.9,
        61.5,
        69.6,
        72.4,
        76.8
      ]
    },
    {
      "model": "GPT-6 Astra",
      "displayName": "GPT-6 Astra",
      "ours": false,
      "metric": "Semantic",
      "values": [
        55.9,
        65.2,
        73.0,
        75.7,
        79.5
      ]
    },
    {
      "model": "GPT-6 Astra",
      "displayName": "GPT-6 Astra",
      "ours": false,
      "metric": "m_tIoU",
      "values": [
        23.3,
        42.7,
        55.7,
        62.7,
        68.1
      ]
    },
    {
      "model": "GPT-6 Astra",
      "displayName": "GPT-6 Astra",
      "ours": false,
      "metric": "Hand",
      "values": [
        67.7,
        76.4,
        83.6,
        86.0,
        88.5
      ]
    }
  ]
}