{ "slug": "vla-robot-benchmarks", "domain": "vlabench.dev", "site_name": "VLA Robotics Benchmark Index", "tagline": "Primary-source index of vision-language-action models, manipulation benchmarks, and robot datasets", "category": "Vision-Language-Action", "updated_at": "2026-10-19", "canonical_definition": "A vision-language-action model maps camera images and a language instruction to robot motor commands. This index records public VLA models, manipulation benchmarks, and training datasets, copying figures only from author papers and project pages retrieved on 19 October 2026. Records include OpenVLA, RT-2, \u03c00, Octo, LIBERO, and Open X-Embodiment.", "meta_description": "Primary-source index of VLA models and manipulation benchmarks, with paper figures for OpenVLA, RT-2, \u03c00, Octo, LIBERO, and Open X-Embodiment.", "headline_stats": [ { "label": "OpenVLA Parameters", "value": "7B", "source": "arXiv:2406.09246" }, { "label": "LIBERO Tasks", "value": "130", "source": "arXiv:2306.03310" }, { "label": "OXE Embodiments", "value": "22", "source": "arXiv:2310.08864" } ], "featured_pairs": [ [ "openvla", "rt-2" ], [ "openvla", "pi0" ], [ "openvla-oft", "smolvla" ], [ "libero", "calvin" ], [ "open-x-embodiment", "droid" ], [ "octo", "rdt-1b" ] ], "columns": [ { "key": "name", "label": "Entity / Implementation" }, { "key": "organization", "label": "Lead Organization" }, { "key": "status", "label": "Maturity / Status" }, { "key": "primary_metric", "label": "Key Technical Metric" }, { "key": "license_or_body", "label": "Governance / License" }, { "key": "confidence", "label": "Verification" } ], "entities": [ { "id": "openvla", "role": "Model", "name": "OpenVLA", "organization": "Stanford / UC Berkeley", "status": "arXiv v3", "primary_metric": "7B; 970k demos", "license_or_body": "Open code and checkpoints", "confidence": "verified", "summary": "7B open VLA trained on 970k robot demonstrations, using Llama 2 with fused DINOv2 and SigLIP features.", "source_url": "https://arxiv.org/html/2406.09246v3", "retrieved_at": "2026-10-19", "specs": { "Parameters": "7B", "Training trajectories": "970k real-world robot demonstrations", "Backbone": "Llama 2 with DINOv2 and SigLIP", "Reported gain": "+16.5% absolute success vs RT-2-X (55B) across 29 tasks", "Fine-tune comparison": "+20.4% vs Diffusion Policy on language-grounded multi-object tasks" } }, { "id": "openvla-oft", "role": "Model", "name": "OpenVLA-OFT", "organization": "Stanford University", "status": "Project release", "primary_metric": "97.1% LIBERO avg", "license_or_body": "Stanford project release", "confidence": "verified", "summary": "Stanford fine-tune of OpenVLA with 97.1% average success on four LIBERO suites and 26x faster actions.", "source_url": "https://openvla-oft.github.io/", "retrieved_at": "2026-10-19", "specs": { "LIBERO average": "97.1% across 4 task suites", "Action speed": "26x faster generation than base OpenVLA", "Recipe": "Parallel decoding, action chunking, continuous actions, L1 regression", "ALOHA variant": "OFT+ adds FiLM for language grounding on a 7B policy" } }, { "id": "rt-2", "role": "Model", "name": "RT-2", "organization": "Google DeepMind", "status": "arXiv 2023", "primary_metric": "Up to 55B; 6k trials", "license_or_body": "Google DeepMind research", "confidence": "verified", "summary": "Google DeepMind VLA that writes robot actions as text tokens, at up to 55B parameters over 6k trials.", "source_url": "https://arxiv.org/abs/2307.15818", "retrieved_at": "2026-10-19", "specs": { "Affiliation": "Google DeepMind", "Scale": "Models with up to 55B parameters", "Evaluation": "6k robotic evaluations", "Action encoding": "Robot actions expressed as text tokens", "Protocol base": "RT-1 data protocol with a vision-language backbone" } }, { "id": "rt-1", "role": "Model", "name": "RT-1", "organization": "Robotics at Google", "status": "RSS 2023", "primary_metric": "97% on 700 instructions", "license_or_body": "Robotics at Google", "confidence": "verified", "summary": "Robotics at Google transformer reporting 97% success on 700 instructions from 130k episodes on 13 robots.", "source_url": "https://doi.org/10.15607/rss.2023.xix.025", "retrieved_at": "2026-10-19", "specs": { "Seen-task success": "97% on over 700 training instructions", "Data collection": "17 months, fleet of 13 robots", "Episodes": "About 130k episodes and over 700 tasks", "Generalization": "New tasks, distractors, and backgrounds 25%, 36%, and 18% better" } }, { "id": "octo", "role": "Model", "name": "Octo", "organization": "UC Berkeley", "status": "arXiv 2024", "primary_metric": "800k trajectories", "license_or_body": "Open pipeline, weights, and data", "confidence": "verified", "summary": "Open transformer policy trained on 800k Open X-Embodiment trajectories and tested across 9 robot platforms.", "source_url": "https://arxiv.org/abs/2405.12213", "retrieved_at": "2026-10-19", "specs": { "Training set": "800k trajectories curated from Open X-Embodiment", "Open X scale noted": "About 1.5M episodes in Open X; RT-X used 350k", "Conditioning": "Language commands or goal images", "Platforms": "Experiments across 9 robotic platforms", "Release": "Training pipeline, checkpoints, and data" } }, { "id": "pi0", "role": "Model", "name": "\u03c00", "organization": "Physical Intelligence", "status": "arXiv preprint", "primary_metric": "7 robots; 68 tasks", "license_or_body": "Physical Intelligence research", "confidence": "verified", "summary": "Flow-matching VLA from Physical Intelligence, pretrained on 7 robot configurations and 68 tasks at up to 50 Hz.", "source_url": "https://arxiv.org/html/2410.24164v4", "retrieved_at": "2026-10-19", "specs": { "Architecture": "Pretrained VLM plus a flow-matching action expert", "Pretraining span": "7 robot configurations and 68 tasks", "Control rate": "Up to 50 Hz", "Embodiments": "Single-arm, dual-arm, and mobile manipulators", "Example tasks": "Laundry folding, table cleaning, box assembly" } }, { "id": "smolvla", "role": "Model", "name": "SmolVLA", "organization": "Hugging Face", "status": "arXiv 2025", "primary_metric": "450M parameters", "license_or_body": "Code, weights, and data released", "confidence": "verified", "summary": "Hugging Face VLA with 450M parameters and a flow-matching action expert, released with code, weights, and data.", "source_url": "https://arxiv.org/html/2506.01844", "retrieved_at": "2026-10-19", "specs": { "Parameters": "450M, about 100M in the action expert", "Backbone": "SmolVLM-2, first 16 LLM layers, VLM frozen", "Action head": "Flow matching, chunks of 50 actions", "Training stack": "LeRobot, 200k pretraining steps, batch size 256", "Release": "Code, pretrained models, and training data" } }, { "id": "groot-n1", "role": "Model", "name": "GR00T N1", "organization": "NVIDIA", "status": "Open foundation model", "primary_metric": "2.2B parameters", "license_or_body": "Publicly released model", "confidence": "verified", "summary": "NVIDIA humanoid VLA. Public GR00T-N1-2B has 2.2B parameters, and the action module runs at 120 Hz.", "source_url": "https://arxiv.org/pdf/2503.14734", "retrieved_at": "2026-10-19", "specs": { "Released checkpoint": "GR00T-N1-2B, 2.2B parameters (1.34B in the VLM)", "System 2": "Eagle-2 VLM at 10 Hz", "System 1": "Diffusion Transformer actions at 120 Hz", "Latency": "63.9 ms for 16 actions on an L40 GPU in bf16", "Robot deployment": "Fourier GR-1 humanoid" } }, { "id": "rdt-1b", "role": "Model", "name": "RDT-1B", "organization": "Tsinghua University", "status": "Project release", "primary_metric": "1.2B; 1M+ episodes", "license_or_body": "Tsinghua research release", "confidence": "verified", "summary": "Tsinghua diffusion transformer with 1.2B parameters, pretrained on 46 datasets and more than 1M episodes.", "source_url": "https://rdt-robotics.github.io/rdt-robotics/", "retrieved_at": "2026-10-19", "specs": { "Parameters": "1.2B", "Pretraining": "46 datasets and 1M+ episodes", "Bimanual fine-tune": "6K+ episodes on the ALOHA dual-arm robot", "Paper id on project page": "arXiv:2410.07864" } }, { "id": "open-x-embodiment", "role": "Dataset", "name": "Open X-Embodiment", "organization": "Open X-Embodiment Collaboration", "status": "Public dataset", "primary_metric": "1M+ traj.; 22 robots", "license_or_body": "RLDS research dataset", "confidence": "verified", "summary": "Cross-lab robot dataset with 1M+ trajectories, 22 embodiments, and 60 source datasets stored as RLDS.", "source_url": "https://arxiv.org/html/2310.08864v1", "retrieved_at": "2026-10-19", "specs": { "Trajectories": "1M+ real-robot trajectories", "Embodiments": "22 robots", "Source datasets": "60 datasets from 34 research labs", "Abstract census": "21 institutions, 527 skills (160266 tasks)", "Format": "RLDS" } }, { "id": "libero", "role": "Benchmark", "name": "LIBERO", "organization": "Bo Liu, Yuke Zhu, Peter Stone et al.", "status": "Public benchmark", "primary_metric": "130 tasks; 4 suites", "license_or_body": "Open benchmark", "confidence": "verified", "summary": "Lifelong manipulation benchmark with 130 language-conditioned tasks in Spatial, Object, Goal, and 100-task suites.", "source_url": "https://arxiv.org/html/2306.03310", "retrieved_at": "2026-10-19", "specs": { "Task count": "130 language-conditioned manipulation tasks", "Suites": "LIBERO-Spatial, LIBERO-Object, LIBERO-Goal, LIBERO-100", "Controlled suites": "Spatial, Object, and Goal have 10 tasks each", "LIBERO-100 split": "90 short-horizon tasks and 10 long-horizon tasks", "Demonstrations": "Human-teleoperated data for all 130 tasks" } }, { "id": "calvin", "role": "Benchmark", "name": "CALVIN", "organization": "University of Freiburg", "status": "Public benchmark", "primary_metric": "4 envs; 20K directives", "license_or_body": "Open-source simulated benchmark", "confidence": "verified", "summary": "Freiburg long-horizon benchmark: 4 indoor environments, about 24 hours of play data, and 20K language directives.", "source_url": "https://arxiv.org/html/2112.03227v4", "retrieved_at": "2026-10-19", "specs": { "Environments": "4 indoor manipulation environments", "Play data": "About 24 hours of teleoperated unstructured play", "Language": "20K language directives", "Control": "7-DOF continuous control", "Zero-shot split": "Train on three environments, test on an unseen scene" } }, { "id": "simplerenv", "role": "Benchmark", "name": "SimplerEnv", "organization": "Li, Hsu, Gu, Mees, Pertsch et al.", "status": "PMLR v270", "primary_metric": "1500+ paired evals", "license_or_body": "Open-sourced environments", "confidence": "verified", "summary": "Real-to-sim policy eval with over 1500 paired sim-and-real trials across two embodiments and eight task families.", "source_url": "https://proceedings.mlr.press/v270/li25c.html", "retrieved_at": "2026-10-19", "specs": { "Paired evaluations": "Over 1500 sim-and-real trials", "Coverage": "Two embodiments and eight task families", "Claim": "Sim performance correlates with real-robot policy behavior", "Project site named in paper": "https://simpler-env.github.io/" } }, { "id": "droid", "role": "Dataset", "name": "DROID", "organization": "DROID collaboration", "status": "Public dataset", "primary_metric": "76k traj.; 350 hours", "license_or_body": "Open dataset and hardware guide", "confidence": "verified", "summary": "In-the-wild Franka dataset: 76k trajectories, 350 hours, 564 scenes, and 86 tasks across 13 institutions.", "source_url": "https://droid-dataset.github.io/", "retrieved_at": "2026-10-19", "specs": { "Trajectories": "76k demonstration trajectories", "Duration": "350 hours", "Diversity": "564 scenes and 86 tasks", "Collection": "50 collectors, 13 institutions, 12 months", "Hardware": "Franka Panda 7DoF, ZED cameras, Quest 2 teleop" } } ], "faqs": [ { "q": "What is a vision-language-action model?", "a": "RT-2 uses the term for a vision-language model co-fine-tuned so robot actions are written as text tokens and trained with web vision-language tasks. At run time those tokens are turned back into motor commands." }, { "q": "How large is OpenVLA, and what was it trained on?", "a": "The OpenVLA paper describes a 7B-parameter model trained on 970k real-world robot demonstrations. It uses a Llama 2 backbone with visual features fused from DINOv2 and SigLIP." }, { "q": "What does the LIBERO benchmark contain?", "a": "LIBERO fixes 130 language-conditioned manipulation tasks in four suites: LIBERO-Spatial, LIBERO-Object, LIBERO-Goal, and LIBERO-100. The first three suites contain 10 tasks each, and LIBERO-100 is split into 90 short-horizon tasks and 10 long-horizon tasks." }, { "q": "How does \u03c00 generate actions?", "a": "The \u03c00 paper adds a flow-matching action expert on a pretrained vision-language model. The reported pretraining mix covers 7 robot configurations and 68 tasks, and the policy can run at up to 50 Hz." }, { "q": "What is in the Open X-Embodiment dataset?", "a": "The Open X-Embodiment paper reports 1M+ real-robot trajectories across 22 embodiments, pooled from 60 datasets and 34 research labs and stored in RLDS. The abstract also counts 527 skills." }, { "q": "How does vlabench.dev check a number?", "a": "Each row cites one author paper or project page retrieved on 2026-10-19. A figure is marked verified when that page states it. Model names, URLs, and scores are not filled in from memory." } ] }