{"manifest":{"schemaVersion":"1.0","pipelineVersion":"benchmark-index-v2","generatedAt":"2026-08-24T07:40:51.872270Z","dataAsOf":"2026-08-24","timezone":"UTC","recordCount":1279,"sourceCoverage":["arXiv OAI-PMH","reviewed official project sources","Hugging Face Hub","GitHub REST"],"isDemo":false,"run":{"sourceDate":"2026-08-23","sourceWindow":{"from":"2026-08-21","to":"2026-08-23"},"papersFetched":0,"accepted":0,"reviewQueued":0},"publicationMetadata":{"observedAt":"2026-08-24T07:42:10.129552Z","source":"arXiv API comments and journal references","recordsChecked":1232,"note":"arXiv venue text is author-provided; official venue adapters may upgrade evidence later."},"metrics":{"observedAt":"2026-08-24T07:42:48.818558Z","attemptedAt":"2026-08-24T07:42:48.818558Z","status":"partial","methodVersion":"attention-ranking-v10","windows":["today","30d","90d"],"note":"Current-level and growth rankings are separate. Growth is missing without a real prior snapshot; negative deltas are preserved.","rerankedFromSnapshot":"2026-08-24","todayObservedAt":"2026-08-24T08:50:50.577661Z","todayAttemptedAt":"2026-08-24T08:50:50.577661Z","todayStatus":"partial"},"latestSourceDate":"2026-08-23","latestBatch":{"from":"2026-08-23","to":"2026-08-23"}},"records":[{"id":"bm_repo-context-ambiguity_46658557","familyId":"bmf_9459abdb9856","name":"repo-context-ambiguity","oneLine":"Benchmark evaluates code generation models on detecting implicit repository conventions. Tasks present a ticket and code context; models must generate code without violating conventions. Automatic AST-based oracles measure constraint violations, security violations, clarification question rate, and functional proxy rate across four context conditions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-23","firstSeenAt":"2026-08-24","recognitionConfidence":0.4,"links":{"report":"https://github.com/Parfait18/repo-context-ambiguity","pdf":null,"project":null,"code":"https://github.com/Parfait18/repo-context-ambiguity","data":null,"hfPaper":null},"evidence":{"snippet":"A benchmark with automatic AST oracles.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:parfait18/repo-context-ambiguity"},"ranking":{"30d":{"score":34,"rank":90,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":313,"coverage":0.55,"confidence":"Low"},"today":{"score":44,"rank":2,"coverage":0.45,"confidence":"Low"}},"description":"Benchmark evaluates code generation models on detecting implicit repository conventions. Tasks present a ticket and code context; models must generate code without violating conventions. Automatic AST-based oracles measure constraint violations, security violations, clarification question rate, and functional proxy rate across four context conditions.","whyItMatters":"Addresses the gap where code LLMs can pass naive tests while violating unstated codebase conventions. Provides a decidable, objective evaluation method for implicit context understanding, useful for improving code assistants in real-world repositories.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"964fd5103e127bb2337f56faa98db2ff5529e67f7b8c59c5f8b50541fba0b46c"},"motivation":"repo-context-ambiguity Do code generation models respect constraints that live in the codebase rather than in the prompt?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"The repository provides a fixed set of 12 tasks, defined oracles, and a clear protocol, allowing other teams to run the same benchmark and compare results."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/Parfait18/repo-context-ambiguity","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:04:32.475961Z"},"evaluationMode":"public_reusable","publishers":[{"name":"Parfait18","organizationType":"community","sourceUrl":"https://github.com/Parfait18/repo-context-ambiguity","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_rag-retrieval-benchmark_2e75b7eb","familyId":"bmf_033fb9287072","name":"rag-retrieval-benchmark","oneLine":"Benchmark evaluates retrieval configurations (dense, BM25, hybrid, reranking) on 648 queries from the FiQA dataset with gold relevance labels. Metrics include recall@10, MRR@10, nDCG@10, and latency. The protocol is deterministic and claims reproducibility to four decimal places.","area":"Language & Knowledge","applicationDomains":["Cybersecurity","Finance & Economics"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity","Financial Services"],"capabilities":["Information retrieval"],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-08-23","firstSeenAt":"2026-08-24","recognitionConfidence":0.4,"links":{"report":"https://github.com/gauthamRohan/rag-retrieval-benchmark","pdf":null,"project":null,"code":"https://github.com/gauthamRohan/rag-retrieval-benchmark","data":"https://huggingface.co/datasets/BeIR/fiqa","hfPaper":null},"evidence":{"snippet":"rag-retrieval-benchmark Does hybrid search and reranking actually help?","reasonCodes":["discovered via github","benchmark term in abstract","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":3235,"hfDatasetLikes":16},"source":{"type":"github","id":"github:gauthamrohan/rag-retrieval-benchmark"},"ranking":{"30d":{"score":41,"rank":89,"coverage":0.7,"confidence":"Medium","datasetDownloadRank":2,"datasetRankPopulation":20},"90d":{"score":42,"rank":262,"coverage":0.85,"confidence":"High","datasetDownloadRank":5,"datasetRankPopulation":51},"today":{"score":44,"rank":3,"coverage":0.5,"confidence":"Medium","datasetDownloadRank":1,"datasetRankPopulation":1}},"description":"Benchmark evaluates retrieval configurations (dense, BM25, hybrid, reranking) on 648 queries from the FiQA dataset with gold relevance labels. Metrics include recall@10, MRR@10, nDCG@10, and latency. The protocol is deterministic and claims reproducibility to four decimal places.","whyItMatters":"Provides evidence on whether hybrid search and reranking improve retrieval quality in real-world settings. The finding that dense retrieval alone can outperform more complex pipelines informs practical RAG system design and avoids unnecessary computational cost.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"6ecf3596337a4eee5b4aab7d310b8dff2f5d4c643ac849c7f889474f0692ceaa"},"motivation":"rag-retrieval-benchmark Does hybrid search and reranking actually help?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"The benchmark uses a fixed public dataset and gold labels, defines clear metrics, and offers code and a notebook for reproduction; results are deterministic."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/gauthamRohan/rag-retrieval-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:04:32.475961Z"},"evaluationMode":"public_reusable","publishers":[{"name":"gauthamRohan","organizationType":"community","sourceUrl":"https://github.com/gauthamRohan/rag-retrieval-benchmark","role":"benchmark-publisher"}],"displayEligible":true,"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"cross-domain"},{"id":"bm_oraclephys_04a2c150","familyId":"bmf_467dce972bf6","name":"OraclePhys","oneLine":"Benchmark evaluates LLM fine-tuning objectives on structural mechanics tasks. Models rank stories of steel frames by inter-story drift and identify the governing story. Evaluation axes include forward ranking, extrapolation, two-step horizons, and post-intervention prediction, with exact finite-element oracle truths.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-23","firstSeenAt":"2026-08-24","recognitionConfidence":0.4,"links":{"report":"https://github.com/cl1110goo-afk/oraclephys","pdf":null,"project":"https://arxiv.org/abs/2608.17162","code":"https://github.com/cl1110goo-afk/oraclephys","data":null,"hfPaper":null},"evidence":{"snippet":"oraclephys Benchmark, datasets and code for arXiv:2608.17162 benchmark finetuning llm reinforcement-learning structural-engineering # OraclePhys An end-to-end instrument — **benchmark, dataset, training pipeline** — for studying what fine-tuning objectives install in LLMs.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:cl1110goo-afk/oraclephys"},"ranking":{"30d":{"score":50,"rank":63,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":272,"coverage":0.55,"confidence":"Low"},"today":{"score":67,"rank":1,"coverage":0.45,"confidence":"Low"}},"description":"Benchmark evaluates LLM fine-tuning objectives on structural mechanics tasks. Models rank stories of steel frames by inter-story drift and identify the governing story. Evaluation axes include forward ranking, extrapolation, two-step horizons, and post-intervention prediction, with exact finite-element oracle truths.","whyItMatters":"Provides a controlled testbed to study how different supervision forms affect LLM capabilities in physics-based domains. The exact oracle allows probing whether models learn underlying principles or surface cues, informing training methodology for scientific applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"76544880e8bcea9bf63a56a109f38763df9d40785c1c34b7fe0fc27e9210d780"},"motivation":"oraclephys Benchmark, datasets and code for arXiv:2608.17162 benchmark finetuning llm reinforcement-learning structural-engineering # OraclePhys An end-to-end instrument — **benchmark, dataset, training pipeline** — for studying what fine-tuning objectives install in LLMs.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"The benchmark ships as self-contained JSON files with prompts and exact oracle answers, a defined parsing and scoring protocol, and code to reproduce all paper results."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/cl1110goo-afk/oraclephys","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:04:32.475961Z"},"evaluationMode":"public_reusable","publishers":[{"name":"cl1110goo-afk","organizationType":"community","sourceUrl":"https://github.com/cl1110goo-afk/oraclephys","role":"benchmark-publisher"}],"displayEligible":true,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_gpu-accelerated-ml-inference-benchmark_af49dfaf","familyId":"bmf_e63546d29f56","name":"GPU-Accelerated-ML-Inference-Benchmark","oneLine":"Tool or script for measuring inference performance across PyTorch CPU, PyTorch CUDA, and TensorRT with a Streamlit dashboard. It is not a repeatable evaluation with a scoring contract.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-23","firstSeenAt":"2026-08-24","recognitionConfidence":0.4,"links":{"report":"https://github.com/jyotydivya/GPU-Accelerated-ML-Inference-Benchmark","pdf":null,"project":null,"code":"https://github.com/jyotydivya/GPU-Accelerated-ML-Inference-Benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"GPU-Accelerated-ML-Inference-Benchmark # GPU-Accelerated ML Inference Benchmark This project demonstrates a benchmark comparing inference performance across different execution engines: 1.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:jyotydivya/gpu-accelerated-ml-inference-benchmark"},"ranking":{"30d":{"score":34,"rank":91,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":314,"coverage":0.55,"confidence":"Low"},"today":{"score":44,"rank":4,"coverage":0.45,"confidence":"Low"}},"description":"Tool or script for measuring inference performance across PyTorch CPU, PyTorch CUDA, and TensorRT with a Streamlit dashboard. It is not a repeatable evaluation with a scoring contract.","whyItMatters":"Demonstrates performance benchmarking but lacks a defined evaluation protocol and comparable scoring, so it does not serve as a standard benchmark for model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"85efadbba086634b6414468a3179dde37711ea68e026bef8107c3c47da75fd2f"},"motivation":"GPU-Accelerated-ML-Inference-Benchmark # GPU-Accelerated ML Inference Benchmark This project demonstrates a benchmark comparing inference performance across different execution engines: 1.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"The repository outlines a repeatable benchmark with defined metrics, batch-size sweep, and a public repository that includes code, results, and deployment instructions."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/jyotydivya/GPU-Accelerated-ML-Inference-Benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:04:32.475961Z"},"evaluationMode":"public_reusable","displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_vnish-global-operator-routing-benchmark_1814945a","familyId":"bmf_17f6ffb52087","name":"vnish-global-operator-routing-benchmark","oneLine":"Benchmark evaluates AI assistants' ability to route operator questions to correct VNISH GLOBAL sources, stop when evidence is incomplete, and avoid unauthorized actions. Contains 80 cases across 10 locales, testing citation precision and rejection of foreign domains.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-24","recognitionConfidence":0.4,"links":{"report":"https://github.com/vnish-global/vnish-global-operator-routing-benchmark","pdf":null,"project":"https://vnish.global/ai/","code":"https://github.com/vnish-global/vnish-global-operator-routing-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"vnish-global-operator-routing-benchmark Multilingual routing and safety benchmark for AI assistants using VNISH GLOBAL owned sources across vnish.global, vnish.ninja and roiasic.com.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:vnish-global/vnish-global-operator-routing-benchmark"},"ranking":{"30d":{"score":34,"rank":92,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":315,"coverage":0.55,"confidence":"Low"}},"description":"Benchmark evaluates AI assistants' ability to route operator questions to correct VNISH GLOBAL sources, stop when evidence is incomplete, and avoid unauthorized actions. Contains 80 cases across 10 locales, testing citation precision and rejection of foreign domains.","whyItMatters":"Addresses evaluation of multilingual routing and safety in AI assistants, which is critical for enterprise deployments with specific source authorities. Provides a structured protocol for testing these capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"45dca0f39a13df0a0a94ef59208a1ce4f78a1c069f1e42a5c6713683509675ca"},"motivation":"vnish-global-operator-routing-benchmark Multilingual routing and safety benchmark for AI assistants using VNISH GLOBAL owned sources across vnish.global, vnish.ninja and roiasic.com.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"The repository specifies data, scoring contract, and release scope, and the code link leads to a public repository containing the benchmark."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/vnish-global/vnish-global-operator-routing-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:04:32.475961Z"},"evaluationMode":"public_reusable","publishers":[{"name":"VNISH GLOBAL","organizationType":"company-research-lab","sourceUrl":"https://github.com/vnish-global/vnish-global-operator-routing-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_vast-benchmarking_a1aad559","familyId":"bmf_6f91daf2a194","name":"vast-benchmarking","oneLine":"A bounded benchmark for measuring hardware capacity of rented GPU containers on Vast.ai, covering GPU throughput, effective CPU concurrency, memory bandwidth, and disk speed, with a fixed 540-second wall-clock budget and normalized JSON output.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/drivelineresearch/vast-benchmarking","pdf":null,"project":"https://cloud.vast.ai/?ref_id=77898","code":"https://github.com/drivelineresearch/vast-benchmarking","data":null,"hfPaper":null},"evidence":{"snippet":"vast-benchmarking Bounded Vast.ai GPU, effective-CPU, memory, and disk benchmark with a SQLite leaderboard benchmark computer-vision cuda flask pytorch sqlite vast-ai Vast Benchmarking Measure the hardware capacity a rented container can actually use.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:drivelineresearch/vast-benchmarking"},"ranking":{"30d":{"score":34,"rank":93,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":316,"coverage":0.55,"confidence":"Low"}},"description":"A bounded benchmark for measuring hardware capacity of rented GPU containers on Vast.ai, covering GPU throughput, effective CPU concurrency, memory bandwidth, and disk speed, with a fixed 540-second wall-clock budget and normalized JSON output.","whyItMatters":"Provides a measurable, reproducible way to compare cloud GPU rental offers based on actual achievable performance rather than advertised specs, enabling more informed purchasing decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"af4cfba93d134407de6f5187f270c1596e6b246e8dd871a5e120ee4129c2e39f"},"motivation":"vast-benchmarking Bounded Vast.ai GPU, effective-CPU, memory, and disk benchmark with a SQLite leaderboard benchmark computer-vision cuda flask pytorch sqlite vast-ai Vast Benchmarking Measure the hardware capacity a rented container can actually use.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"Defines a repeatable bounded benchmark with a stable scoring methodology, portable JSON outputs, SQLite rankings, and a public dashboard, along with code and documentation for others to run."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/drivelineresearch/vast-benchmarking","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"score_submission","publishers":[{"name":"Driveline Research","organizationType":"academic-lab","sourceUrl":"https://github.com/drivelineresearch/vast-benchmarking","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_tw-exam-bench_0b096fe2","familyId":"bmf_2fc7081321fe","name":"tw-exam-bench","oneLine":"Benchmark for Taiwanese professional exams, containing multiple-choice questions across various fields like medicine, law, and engineering. Configurations per field, with test splits. Scored by accuracy on multiple-choice answers.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-24","recognitionConfidence":0.4,"links":{"report":"https://huggingface.co/datasets/OpenTWBench/tw-exam-bench","pdf":null,"project":null,"code":null,"data":"https://huggingface.co/datasets/OpenTWBench/tw-exam-bench","hfPaper":null},"evidence":{"snippet":"🔗 opentwbench.ai · leaderboard, methodology, submissions from datasets import load_dataset ds = load_dataset(\"OpenTWBench/tw-exam-bench\", \"dentistry\", split=\"test\") ds = load_dataset(\"OpenTWBench/tw-exam-bench\", \"field_medicine\", split=\"test\") ds =… See the full description on the dataset page: https://huggingface.co/datasets/OpenTWBench/tw-exam-bench.","reasonCodes":["discovered via huggingface","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":62,"hfDatasetLikes":0},"source":{"type":"huggingface","id":"huggingface:opentwbench/tw-exam-bench"},"ranking":{"30d":{"score":49,"rank":80,"coverage":0.15,"confidence":"Low","datasetDownloadRank":12,"datasetRankPopulation":20},"90d":{"score":44,"rank":238,"coverage":0.3,"confidence":"Low","datasetDownloadRank":36,"datasetRankPopulation":51}},"description":"Benchmark for Taiwanese professional exams, containing multiple-choice questions across various fields like medicine, law, and engineering. Configurations per field, with test splits. Scored by accuracy on multiple-choice answers.","whyItMatters":"Provides a broad, real-world evaluation set for Chinese language models in professional domains, useful for assessing knowledge and reasoning capabilities in specific fields.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"9d2756a025b4fce4d4c6cabaafae4e2c71308a90bec52669d0ddae875b26e8ea"},"motivation":"OpenTWBench 255,854 multiple-choice questions across 148 academic subjects, measuring what a language model knows about Taiwan — in Traditional Chinese, from the national examinations that license Taiwan's regulated professions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"The dataset is available on Hugging Face with named configurations and an explicit linkage to a leaderboard, methodology, and submissions, indicating a credible public reuse path."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://huggingface.co/datasets/OpenTWBench/tw-exam-bench","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:04:32.475961Z"},"evaluationMode":"public_reusable","publishers":[{"name":"OpenTWBench","organizationType":"benchmark-organization","sourceUrl":"https://huggingface.co/datasets/OpenTWBench/tw-exam-bench","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_tw-bench_62d8b3a2","familyId":"bmf_be1e92b1a963","name":"tw-bench","oneLine":"A multiple-choice question-answering benchmark for Traditional Chinese (Taiwan) across 17 professional fields, with over 100k test items derived from certification exams, scored by exact match.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://huggingface.co/datasets/OpenTWBench/tw-bench","pdf":null,"project":null,"code":null,"data":"https://huggingface.co/datasets/OpenTWBench/tw-bench","hfPaper":null},"evidence":{"snippet":"🔗 opentwbench.ai · leaderboard, methodology, submissions from datasets import load_dataset ds = load_dataset(\"OpenTWBench/tw-bench\", \"dentistry\", split=\"test\") ds = load_dataset(\"OpenTWBench/tw-bench\", \"field_medicine\", split=\"test\") ds =… See the full description on the dataset page: https://huggingface.co/datasets/OpenTWBench/tw-bench.","reasonCodes":["discovered via huggingface","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":62,"hfDatasetLikes":0},"source":{"type":"huggingface","id":"huggingface:opentwbench/tw-bench"},"ranking":{"30d":{"score":49,"rank":81,"coverage":0.15,"confidence":"Low","datasetDownloadRank":13,"datasetRankPopulation":20},"90d":{"score":44,"rank":239,"coverage":0.3,"confidence":"Low","datasetDownloadRank":37,"datasetRankPopulation":51}},"description":"A multiple-choice question-answering benchmark for Traditional Chinese (Taiwan) across 17 professional fields, with over 100k test items derived from certification exams, scored by exact match.","whyItMatters":"Fills a gap in Traditional Chinese evaluation, offering a large-scale, domain-specific benchmark for assessing LLM knowledge in professional contexts with a clear scoring protocol.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"cde788f0b2f6f81d27e20aa5879cb96314def8e35ae2cb77eef74e31607a6147"},"motivation":"OpenTWBench 255,854 multiple-choice questions across 148 academic subjects, measuring what a language model knows about Taiwan — in Traditional Chinese, from the national examinations that license Taiwan's regulated professions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"Provides a large, clearly defined multiple-choice dataset with official configs and intended leaderboard/submission usage, offering a stable scoring contract and public reuse path."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://huggingface.co/datasets/OpenTWBench/tw-bench","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"score_submission","publishers":[{"name":"OpenTWBench","organizationType":"benchmark-organization","sourceUrl":"https://huggingface.co/datasets/OpenTWBench/tw-bench","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_turkish-llm-benchmark_fa8945d1","familyId":"bmf_ddbc317711bc","name":"turkish-llm-benchmark","oneLine":"A provider-neutral benchmark suite for evaluating Turkish LLMs on quality, instruction following, latency, cost, and throughput, with 12 tasks and an OpenAI-compatible interface.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/kadirnar/turkish-llm-benchmark","pdf":null,"project":"https://docs.astral.sh/uv/","code":"https://github.com/kadirnar/turkish-llm-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"turkish-llm-benchmark Provider-neutral Turkish quality, cost, latency, and performance benchmarks for LLMs benchmark evaluation llm openrouter turkish # Turkish LLM Benchmark [![CI](https://github.com/kadirnar/turkish-llm-benchmark/actions/workflows/ci.yml/badge.svg)](https://github.com/kadirnar/turkish-llm-benchmark/actions/workflows/ci.yml) A provider-neutral, reproducible benchmark suite for measuring Turkish language quality, instruction following, latency, API cost, local throughput, and sp","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:kadirnar/turkish-llm-benchmark"},"ranking":{"30d":{"score":50,"rank":64,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":273,"coverage":0.55,"confidence":"Low"}},"description":"A provider-neutral benchmark suite for evaluating Turkish LLMs on quality, instruction following, latency, cost, and throughput, with 12 tasks and an OpenAI-compatible interface.","whyItMatters":"Addresses the lack of standardized Turkish LLM evaluation, offering a reproducible protocol for comparing models across providers on both quality and efficiency metrics.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"252552faa609b9909f4e49ff9a5fc7d29a5802291d68254d3faee4a93a003332"},"motivation":"turkish-llm-benchmark Provider-neutral Turkish quality, cost, latency, and performance benchmarks for LLMs benchmark evaluation llm openrouter turkish # Turkish LLM Benchmark [![CI](https://github.com/kadirnar/turkish-llm-benchmark/actions/workflows/ci.yml/badge.svg)](https://github.com/kadirnar/turkish-llm-benchmark/actions/workflows/ci.yml) A provider-neutral, reproducible benchmark suite for measuring Turkish lan…","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"Defines a stable task set, scoring metrics, and a public CLI for running evaluations against any compatible endpoint, with OpenRouter support and documented configuration."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/kadirnar/turkish-llm-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"score_submission","publishers":[{"name":"Kadir Nar","organizationType":"community","sourceUrl":"https://github.com/kadirnar/turkish-llm-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_russian-it-community-corpus_d8ce4257","familyId":"bmf_399000b287dd","name":"Russian IT Community Corpus","oneLine":"A large-scale, de-identified corpus of Russian IT community discussions (2.91M messages) with SFT dialogues, DPO pairs, and RAG knowledge base chunks for fine-tuning and evaluation of Russian-language LLMs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-24","recognitionConfidence":0.4,"links":{"report":"https://github.com/wwewtech/russian-it-community-corpus","pdf":null,"project":"https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Dataset-FFD21E?style=flat-square","code":"https://github.com/wwewtech/russian-it-community-corpus","data":"https://huggingface.co/datasets/wwewtech/russian-it-community-corpus","hfPaper":null},"evidence":{"snippet":"russian-it-community-corpus Russian IT Community Conversational Corpus (2018-2026) · Zero-PII Curation Platform, Multi-turn SFT, DPO, RAG Knowledge Base, Streamlit Studio & RTX 3060 LoRA benchmark data-engineering dataset dpo llm lora machine-learning rag russian-nlp sft streamlit zero-pii **High-throughput data engineering and Zero-PII curation platform for language models** 2.91M+ discussions · 2017–2026 history · SFT dialogues · DPO pairs · RAG knowledge base · LoRA on RTX 3060 [![Hugging Fac","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":22,"hfDatasetLikes":0},"source":{"type":"github","id":"github:wwewtech/russian-it-community-corpus"},"ranking":{"30d":{"score":28,"rank":140,"coverage":0.7,"confidence":"Medium","datasetDownloadRank":19,"datasetRankPopulation":20},"90d":{"score":16,"rank":396,"coverage":0.85,"confidence":"High","datasetDownloadRank":48,"datasetRankPopulation":51}},"description":"A large-scale, de-identified corpus of Russian IT community discussions (2.91M messages) with SFT dialogues, DPO pairs, and RAG knowledge base chunks for fine-tuning and evaluation of Russian-language LLMs.","whyItMatters":"Provides a publicly available, privacy-curated resource for training and benchmarking Russian-language models in technical domains, enabling reproducibility and comparison across fine-tuning and retrieval tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T08:57:07.998303Z","inputHash":"a0237c031ff0e5858d209ec12d5209844376fe9bd9476d27334d993fbbd2e6da"},"motivation":"russian-it-community-corpus Russian IT Community Conversational Corpus (2018-2026) · Zero-PII Curation Platform, Multi-turn SFT, DPO, RAG Knowledge Base, Streamlit Studio & RTX 3060 LoRA benchmark data-engineering dataset dpo llm lora machine-learning rag russian-nlp sft streamlit zero-pii **High-throughput data engineering and Zero-PII curation platform for language models** 2.91M+ discussions · 2017–2026 history ·…","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:41:28.683304Z","model":"deepseek-v4-pro","decisionReason":"Named corpus with released data, code, and documented evaluation metrics; usable by other teams for reproducible modeling."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/wwewtech/russian-it-community-corpus","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:40:52.396154Z"},"evaluationMode":"public_reusable","publishers":[{"name":"wwewtech","organizationType":"community","sourceUrl":"https://github.com/wwewtech/russian-it-community-corpus","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_roman-numeral-harmony-benchmark_f8f54472","familyId":"bmf_4a091afad75c","name":"roman-numeral-harmony-benchmark","oneLine":"A synthetic benchmark for functional harmony analysis, evaluating Roman numeral transcription, cadence classification, and key identification across four input representations, with exact-match scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/4esv/roman-numeral-harmony-benchmark","pdf":null,"project":"https://huggingface.co/datasets/4esv/roman-numeral-harmony-benchmark","code":"https://github.com/4esv/roman-numeral-harmony-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"roman-numeral-harmony-benchmark A benchmark for functional harmony: Roman-numeral analysis, cadence classification and key identification, in four representations.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:4esv/roman-numeral-harmony-benchmark"},"ranking":{"30d":{"score":34,"rank":94,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":317,"coverage":0.55,"confidence":"Low"}},"description":"A synthetic benchmark for functional harmony analysis, evaluating Roman numeral transcription, cadence classification, and key identification across four input representations, with exact-match scoring.","whyItMatters":"Provides a deterministic, grammar-based test for music-theory reasoning in LLMs, filling a gap in evaluating symbolic music understanding and offering a clear scoring contract.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"af3fb680c4ad209a365b75c611efc06f90e5c43f43b39e046380ea7f899d30e9"},"motivation":"roman-numeral-harmony-benchmark A benchmark for functional harmony: Roman-numeral analysis, cadence classification and key identification, in four representations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"Offers a fixed benchmark dataset with exact-match scoring, a public evaluation harness, and documented results from multiple models, making it suitable for ongoing model comparison."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/4esv/roman-numeral-harmony-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"score_submission","publishers":[{"name":"4esv","organizationType":"community","sourceUrl":"https://github.com/4esv/roman-numeral-harmony-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_refusal-benchmark_83307c24","familyId":"bmf_076cfc84d10b","name":"refusal-benchmark","oneLine":"A tool and prompt set for measuring how often LLM chat endpoints refuse harmful prompts, classifying responses into refusal, compliance, empty, or truncated with a defined effective refusal rate.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/audn-ai/refusal-benchmark","pdf":null,"project":"https://audn.ai/audncode","code":"https://github.com/audn-ai/refusal-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"refusal-benchmark 519+ harmful prompts to detect how abliterated AI models are # refusal-benchmark > 519+ harmful prompts to detect how abliterated AI models are.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:audn-ai/refusal-benchmark"},"ranking":{"30d":{"score":50,"rank":65,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":274,"coverage":0.55,"confidence":"Low"}},"description":"A tool and prompt set for measuring how often LLM chat endpoints refuse harmful prompts, classifying responses into refusal, compliance, empty, or truncated with a defined effective refusal rate.","whyItMatters":"Offers a practical, reproducible method for evaluating safety alignment and refusal behavior across different model deployments, which is crucial for responsible AI deployment decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"60c49ef7d331a84c0d66ad7231c85d14f6f80733444c6ce0eecde2b9bcb19132"},"motivation":"refusal-benchmark 519+ harmful prompts to detect how abliterated AI models are # refusal-benchmark > 519+ harmful prompts to detect how abliterated AI models are.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"The repository defines a repeatable prompt set, response classification rubric, and effective refusal metric, with code and example results publicly available for other teams to run against their own endpoints."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/audn-ai/refusal-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"score_submission","publishers":[{"name":"Audn AI","organizationType":"company-research-lab","sourceUrl":"https://github.com/audn-ai/refusal-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_mandate-bench_a638082e","familyId":"bmf_692dbb1815c5","name":"mandate-bench","oneLine":"A behavioral benchmark for LLM agents on a portfolio-rebalancing task, measuring mandate compliance, decision consistency, and reasoning-action agreement, with a clear scoring rubric.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/msavdert/mandate-bench","pdf":null,"project":"https://msavdert.github.io/mandate-bench/","code":"https://github.com/msavdert/mandate-bench","data":null,"hfPaper":null},"evidence":{"snippet":"mandate-bench Behavioral benchmarking of LLM agents on a fixed portfolio-rebalancing task: mandate compliance, decision consistency, reasoning-action agreement agents benchmark instruction-following llm reproducibility # Mandate Bench Behavioral benchmarking of LLM agents on a fixed portfolio-rebalancing task: mandate compliance, decision consistency, and reasoning-action agreement.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:msavdert/mandate-bench"},"ranking":{"30d":{"score":34,"rank":95,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":318,"coverage":0.55,"confidence":"Low"}},"description":"A behavioral benchmark for LLM agents on a portfolio-rebalancing task, measuring mandate compliance, decision consistency, and reasoning-action agreement, with a clear scoring rubric.","whyItMatters":"Provides a controlled environment to evaluate agent behavior beyond profit, focusing on rule adherence and reliability, which is essential for deploying agents in regulated domains.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"f143798431dae510a84d4b94a3fd3b350b0243d5a04757be0ff072743b47716f"},"motivation":"mandate-bench Behavioral benchmarking of LLM agents on a fixed portfolio-rebalancing task: mandate compliance, decision consistency, reasoning-action agreement agents benchmark instruction-following llm reproducibility # Mandate Bench Behavioral benchmarking of LLM agents on a fixed portfolio-rebalancing task: mandate compliance, decision consistency, and reasoning-action agreement.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"The repository defines a frozen task, explicit scoring metrics, and a public leaderboard with code and results, allowing other teams to replicate the evaluation and compare new agents."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/msavdert/mandate-bench","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"public_reusable","publishers":[{"name":"msavdert","organizationType":"community","sourceUrl":"https://github.com/msavdert/mandate-bench","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_llm-finetuning-benchmark_efda9d1c","familyId":"bmf_663a79d7b514","name":"LLM Fine-tuning Benchmark","oneLine":"Benchmarks different fine-tuning strategies (full fine-tuning, LoRA, QLoRA) on Financial PhraseBank using Qwen 2.5-0.5B, evaluating performance vs cost and memory.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-24","recognitionConfidence":0.4,"links":{"report":"https://github.com/Ishant-005/llm-finetuning-benchmark","pdf":null,"project":null,"code":"https://github.com/Ishant-005/llm-finetuning-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"llm-finetuning-benchmark Benchmarked different fine tuning strategies like full ft, LoRA, QLoRA.","reasonCodes":["discovered via github","benchmark term in abstract","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:ishant-005/llm-finetuning-benchmark"},"ranking":{"30d":{"score":34,"rank":96,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":319,"coverage":0.55,"confidence":"Low"}},"description":"Benchmarks different fine-tuning strategies (full fine-tuning, LoRA, QLoRA) on Financial PhraseBank using Qwen 2.5-0.5B, evaluating performance vs cost and memory.","whyItMatters":"Could inform selection of fine-tuning methods for resource-constrained settings, but lacks a formal evaluation contract and public scoring mechanism.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T08:57:07.998303Z","inputHash":"a635218fc1001f80fea1d469dd7ff455b9a0f64bffb8611e14aef1739c64b06e"},"motivation":"llm-finetuning-benchmark Benchmarked different fine tuning strategies like full ft, LoRA, QLoRA.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:41:28.683304Z","model":"deepseek-v4-pro","decisionReason":"Public repository with benchmarking methodology and open dataset enables reproducible comparisons."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/Ishant-005/llm-finetuning-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:40:52.396154Z"},"evaluationMode":"public_reusable","displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_groundtruth-dynamic-benchmarking_4f938e9e","familyId":"bmf_d6fd851014b2","name":"groundtruth-dynamic-benchmarking","oneLine":"A geology question-answering benchmark with grounding rubrics and evidence tracing, evaluating LLMs on real-world geological reasoning via LLM judge scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://huggingface.co/datasets/EigenformAI/groundtruth-dynamic-benchmarking","pdf":null,"project":null,"code":null,"data":"https://huggingface.co/datasets/EigenformAI/groundtruth-dynamic-benchmarking","hfPaper":null},"evidence":{"snippet":"Groundtruth Dynamic Benchmarking — Geology Question sets and grading rubrics for evaluating LLMs on real-world geological reasoning.","reasonCodes":["discovered via huggingface","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":78,"hfDatasetLikes":0},"source":{"type":"huggingface","id":"huggingface:eigenformai/groundtruth-dynamic-benchmarking"},"ranking":{"30d":{"score":51,"rank":61,"coverage":0.15,"confidence":"Low","datasetDownloadRank":9,"datasetRankPopulation":20},"90d":{"score":46,"rank":229,"coverage":0.3,"confidence":"Low","datasetDownloadRank":33,"datasetRankPopulation":51}},"description":"A geology question-answering benchmark with grounding rubrics and evidence tracing, evaluating LLMs on real-world geological reasoning via LLM judge scoring.","whyItMatters":"Fills a gap in domain-specific scientific evaluation, offering a transparent scoring rubric and evidence-based grading to assess LLM reasoning in professional geology contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"6a289a823e314bfb8e6e2b7852f93eff9e0bedc90d35197ca18bbc402d3b2cdb"},"motivation":"Groundtruth Dynamic Benchmarking — Geology Question sets and grading rubrics for evaluating LLMs on real-world geological reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"The benchmark defines a clear task, a stable rubric-based scoring contract, and a public path via the dataset and GitHub harness with a community leaderboard."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://huggingface.co/datasets/EigenformAI/groundtruth-dynamic-benchmarking","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"score_submission","publishers":[{"name":"Eigenform AI","organizationType":"company-research-lab","sourceUrl":"https://huggingface.co/datasets/EigenformAI/groundtruth-dynamic-benchmarking","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_grit-benchmark_0cf59789","familyId":"bmf_39727565f571","name":"grit-benchmark","oneLine":"Benchmark evaluates LLM planners that drive robot skills in simulated environments with disruptions (object teleport, grip failures). Tasks involve recovery after plan breaks. Metrics include success rate and recovery rate across turbulence tiers, with seeds for reproducibility.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-24","recognitionConfidence":0.4,"links":{"report":"https://github.com/guptabhishekumar/grit-benchmark","pdf":null,"project":"https://mujoco.org","code":"https://github.com/guptabhishekumar/grit-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"grit-benchmark GRIT: a benchmark scoring LLM planners that drive robot skills while the world undoes their work.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:guptabhishekumar/grit-benchmark"},"ranking":{"30d":{"score":34,"rank":97,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":320,"coverage":0.55,"confidence":"Low"}},"description":"Benchmark evaluates LLM planners that drive robot skills in simulated environments with disruptions (object teleport, grip failures). Tasks involve recovery after plan breaks. Metrics include success rate and recovery rate across turbulence tiers, with seeds for reproducibility.","whyItMatters":"Tests robustness and recovery capabilities of AI planners in dynamic real-world conditions, which is crucial for deploying robots in uncertain environments. Provides a standardized simulation for comparative evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"885e8106116c912c638274bec531f74d11d890971d3e62e511e1257e22a3b4aa"},"motivation":"grit-benchmark GRIT: a benchmark scoring LLM planners that drive robot skills while the world undoes their work.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"GRIT provides a clear scoring contract, reproducible turbulence schedules, and a public code repository with committed baseline logs and a leaderboard generation tool."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/guptabhishekumar/grit-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:04:32.475961Z"},"evaluationMode":"public_reusable","publishers":[{"name":"guptabhishekumar","organizationType":"community","sourceUrl":"https://github.com/guptabhishekumar/grit-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_frontier-agent-benchmark_f8aadff6","familyId":"bmf_fd79a742d17a","name":"frontier-agent-benchmark","oneLine":"A benchmark for evaluating autonomous AI engineering agents on engineering quality dimensions, with observed and estimated telemetry and a weighted scoring model.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/shubhraj5575/frontier-agent-benchmark","pdf":null,"project":"http://localhost:8737","code":"https://github.com/shubhraj5575/frontier-agent-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"frontier-agent-benchmark # Frontier Agent Benchmark (FAB) **Independent observability and benchmarking platform for evaluating autonomous AI engineering agents on engineering quality - not volume.** FAB answers a different question than \"how much code did the agent write?\".","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:shubhraj5575/frontier-agent-benchmark"},"ranking":{"30d":{"score":34,"rank":98,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":321,"coverage":0.55,"confidence":"Low"}},"description":"A benchmark for evaluating autonomous AI engineering agents on engineering quality dimensions, with observed and estimated telemetry and a weighted scoring model.","whyItMatters":"Provides a holistic assessment of agent-built software quality beyond code volume, with transparent provenance tracking, aiding in agent capability evaluation and improvement.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"462dcef720a83e964cb5db2ed4274285e46b72ff13c711ce2b2a1501c6ebaec8"},"motivation":"frontier-agent-benchmark # Frontier Agent Benchmark (FAB) **Independent observability and benchmarking platform for evaluating autonomous AI engineering agents on engineering quality - not volume.** FAB answers a different question than \"how much code did the agent write?\".","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"The repository provides a fully specified scoring engine with defined weights and formulas, a public codebase, and clear instructions for running the benchmark on custom agent-built projects, as well as a live results dashboard."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/shubhraj5575/frontier-agent-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"score_submission","publishers":[{"name":"Shubhraj5575","organizationType":"community","sourceUrl":"https://github.com/shubhraj5575/frontier-agent-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_from-financial-sentiment-classification-to_65cf9b8b","familyId":"bmf_322d40bddebf","name":"From Financial Sentiment Classification to Return Predictability: A QLoRA Benchmark of Large Language Models","oneLine":"Benchmark evaluating QLoRA-tuned LLMs on financial sentiment classification and return predictability using datasets like Financial PhraseBank and Twitter Financial News Sentiment, with metrics including accuracy, Rank IC, and portfolio returns.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-24","recognitionConfidence":0.4,"links":{"report":"https://github.com/chirindaopensource/from_financial_sentiment_classification_to_return_predictability","pdf":null,"project":"https://img.shields.io/badge/License-MIT-2C3E50?style=flat","code":"https://github.com/chirindaopensource/from_financial_sentiment_classification_to_return_predictability","data":"https://huggingface.co/datasets/financial_phrasebank","hfPaper":null},"evidence":{"snippet":"from_financial_sentiment_classification_to_return_predictability End-to-End Python implementation of Luo's (2026) benchmark construction and evaluation stack for financial NLP.","reasonCodes":["discovered via github","benchmark term in abstract","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:chirindaopensource/from_financial_sentiment_classification_to_return_predictability"},"ranking":{"30d":{"score":34,"rank":99,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":322,"coverage":0.55,"confidence":"Low"}},"description":"Benchmark evaluating QLoRA-tuned LLMs on financial sentiment classification and return predictability using datasets like Financial PhraseBank and Twitter Financial News Sentiment, with metrics including accuracy, Rank IC, and portfolio returns.","whyItMatters":"Connects NLP model performance to economic outcomes, offering a practical evaluation for financial applications that goes beyond classification accuracy.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T08:57:07.998303Z","inputHash":"237f3b226725bf6087dd1c40bbbf69644ee72674d0c3d265d2e549357ab47f46"},"motivation":"from_financial_sentiment_classification_to_return_predictability End-to-End Python implementation of Luo's (2026) benchmark construction and evaluation stack for financial NLP.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:41:28.683304Z","model":"deepseek-v4-pro","decisionReason":"End-to-end implementation with code and data sources allows independent replication of the benchmark and economic evaluation."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/chirindaopensource/from_financial_sentiment_classification_to_return_predictability","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:40:52.396154Z"},"evaluationMode":"public_reusable","publishers":[{"name":"Craig Chirinda (Open Source Projects)","organizationType":"community","sourceUrl":"https://github.com/chirindaopensource/from_financial_sentiment_classification_to_return_predictability","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_evallab_50a8738a","familyId":"bmf_5ed793692e46","name":"evallab","oneLine":"EvalLab is a research-style evaluation project for local LLMs that studies how benchmark size, composition, measurement ambiguity, generation variance, and prompt framing affect confidence in apparent model rankings. It provides a paired benchmark with 120 deterministic Python coding tasks and reports primary results, including a ranking reversal and bootstrap uncertainty analysis.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/bagheri365/evallab","pdf":null,"project":null,"code":"https://github.com/bagheri365/evallab","data":null,"hfPaper":null},"evidence":{"snippet":"evallab Research-style evaluation of local LLMs focused on paired comparisons, uncertainty, benchmark stability, and trustworthy model-ranking conclusions.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:bagheri365/evallab"},"ranking":{"30d":{"score":34,"rank":100,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":323,"coverage":0.55,"confidence":"Low"}},"description":"EvalLab is a research-style evaluation project for local LLMs that studies how benchmark size, composition, measurement ambiguity, generation variance, and prompt framing affect confidence in apparent model rankings. It provides a paired benchmark with 120 deterministic Python coding tasks and reports primary results, including a ranking reversal and bootstrap uncertainty analysis.","whyItMatters":"EvalLab addresses the evaluation gap of understanding when observed differences between AI systems are trustworthy, offering practical decision value for researchers and practitioners who need to interpret model comparisons reliably. It highlights that benchmark construction can influence rankings more than generation variability or prompt changes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"28f2790964760499f78e565822ec8a4729aaf7a7c6398b80db6f7d99fefd68c7"},"motivation":"evallab Research-style evaluation of local LLMs focused on paired comparisons, uncertainty, benchmark stability, and trustworthy model-ranking conclusions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"The benchmark is defined, frozen, and scored with hidden pytest tests, and the code and benchmark are publicly available on GitHub for others to run or inspect."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/bagheri365/evallab","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"score_submission","displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_delverl_572c8749","familyId":"bmf_0c3fd3592789","name":"DelveRL","oneLine":"DelveRL is a human-playable, turn-based roguelike and open benchmark for local game-playing agents. It provides a deterministic, procedurally generated, partially observed environment with a standardized API, a released baseline model, and benchmark audits with scoring based on floor progression.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/SnyderConsulting/DelveRL","pdf":null,"project":"https://godotengine.org/download/","code":"https://github.com/SnyderConsulting/DelveRL","data":null,"hfPaper":null},"evidence":{"snippet":"DelveRL A human-playable turn-based roguelike and open benchmark for local game-playing agents.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:snyderconsulting/delverl"},"ranking":{"30d":{"score":50,"rank":66,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":275,"coverage":0.55,"confidence":"Low"}},"description":"DelveRL is a human-playable, turn-based roguelike and open benchmark for local game-playing agents. It provides a deterministic, procedurally generated, partially observed environment with a standardized API, a released baseline model, and benchmark audits with scoring based on floor progression.","whyItMatters":"DelveRL fills a gap for a reproducible and accessible benchmark for game-playing agents, offering a structured environment with clear evaluation criteria. It enables comparison of agent performance in a complex, partially observable setting, supporting progress in reinforcement learning and game AI.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"a88f3120135c911954811bbdfe7bd4843007e2383a0cd5c7e9e32e8e2a41b381"},"motivation":"DelveRL A human-playable turn-based roguelike and open benchmark for local game-playing agents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"The repository includes a complete game, API, training tools, released baseline checkpoint, and raw benchmark audits, with explicit documentation for evaluation protocol and results, enabling ongoing model comparison and public reuse."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/SnyderConsulting/DelveRL","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"score_submission","publishers":[{"name":"Snyder Consulting","organizationType":"company-research-lab","sourceUrl":"https://github.com/SnyderConsulting/DelveRL","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_ct-segmentation-benchmark_d7d89501","familyId":"bmf_2a58534eb608","name":"ct-segmentation-benchmark","oneLine":"A reproducible benchmark comparing three public automated organ-segmentation tools (MOOSE, TotalSegmentator, VoxTell) on public CT datasets, scored with Dice and IoU across up to 16 organs. The benchmark uses a staged methodology and provides scripts for running tools, discovering organs, and evaluating results.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Software & Cloud","Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/caitlin-leonard/ct-segmentation-benchmark","pdf":null,"project":"https://drive.google.com/file/d/1jloIo4DEvmwkRSWtV7dEShO3bL9ReBie/view?usp=sharing","code":"https://github.com/caitlin-leonard/ct-segmentation-benchmark","data":"https://zenodo.org/record/6802614","hfPaper":null},"evidence":{"snippet":"ct-segmentation-benchmark Staged Dice/IoU benchmark of MOOSE, TotalSegmentator & VoxTell on public CT datasets.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:caitlin-leonard/ct-segmentation-benchmark"},"ranking":{"30d":{"score":34,"rank":101,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":324,"coverage":0.55,"confidence":"Low"}},"description":"A reproducible benchmark comparing three public automated organ-segmentation tools (MOOSE, TotalSegmentator, VoxTell) on public CT datasets, scored with Dice and IoU across up to 16 organs. The benchmark uses a staged methodology and provides scripts for running tools, discovering organs, and evaluating results.","whyItMatters":"This benchmark addresses the evaluation gap of comparing segmentation tools under controlled conditions, providing a public and reproducible protocol. It offers practical value for researchers selecting segmentation tools by showing how conclusions can change with sample size and organ set, emphasizing the need for robust evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"727af7fda73ae0434421461b974670b9a7e3f8b48301b6288954c338ada9aa58"},"motivation":"ct-segmentation-benchmark Staged Dice/IoU benchmark of MOOSE, TotalSegmentator & VoxTell on public CT datasets.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"Naming a specific evaluation object (three tools vs. ground truth on FLARE22 and TotalSegmentator dataset), defining a scoring contract (Dice/IoU per organ subject, fixed seed sample), and releasing code and protocol publicly. README explicitly states it is a benchmark with reproducible scripts."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/caitlin-leonard/ct-segmentation-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"public_reusable","publishers":[{"name":"Caitlin Leonard","organizationType":"academic-lab","sourceUrl":"https://github.com/caitlin-leonard/ct-segmentation-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Multimodal Perception","Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_cross-engine-humanoid-benchmark_f797b695","familyId":"bmf_0ef7a688a656","name":"cross-engine-humanoid-benchmark","oneLine":"A cross-engine benchmark (MuJoCo vs PyBullet) for a Unitree G1 humanoid robot, applying a standardized squat-and-recover trajectory and measuring tracking RMSE, CoM deviation, contact forces, torque, and computational speed. Includes PSO-based calibration for sim-to-real transfer.","area":"Language & Knowledge","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.AI"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/tiago369/cross-engine-humanoid-benchmark","pdf":null,"project":null,"code":"https://github.com/tiago369/cross-engine-humanoid-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"cross-engine-humanoid-benchmark Cross-engine (MuJoCo vs PyBullet) humanoid benchmark with PSO-based sim calibration # Cross-Engine Humanoid Benchmark [![benchmark](https://github.com/tiago369/cross-engine-humanoid-benchmark/actions/workflows/benchmark.yml/badge.svg)](https://github.com/tiago369/cross-engine-humanoid-benchmark/actions/workflows/benchmark.yml) **Build, configure, and operate a robotics simulation environment; run a standardized benchmark against a scripted reference task; evaluate","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:tiago369/cross-engine-humanoid-benchmark"},"ranking":{"30d":{"score":34,"rank":102,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":325,"coverage":0.55,"confidence":"Low"}},"description":"A cross-engine benchmark (MuJoCo vs PyBullet) for a Unitree G1 humanoid robot, applying a standardized squat-and-recover trajectory and measuring tracking RMSE, CoM deviation, contact forces, torque, and computational speed. Includes PSO-based calibration for sim-to-real transfer.","whyItMatters":"This benchmark provides a reproducible method for comparing physics simulators on the same robot and task, highlighting engine-induced differences. It supports simulator selection, sim-to-real transfer research, and understanding the impact of engine choice on robot control evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T06:57:28.889278Z","inputHash":"8f2ad715f587aa9153b703788b05650b00a7b4bb8f4124d6c2a011922d2d8906"},"motivation":"cross-engine-humanoid-benchmark Cross-engine (MuJoCo vs PyBullet) humanoid benchmark with PSO-based sim calibration # Cross-Engine Humanoid Benchmark [![benchmark](https://github.com/tiago369/cross-engine-humanoid-benchmark/actions/workflows/benchmark.yml/badge.svg)](https://github.com/tiago369/cross-engine-humanoid-benchmark/actions/workflows/benchmark.yml) **Build, configure, and operate a robotics simulation envi…","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T05:47:18.116919Z","model":"deepseek-v4-pro","decisionReason":"The repository provides a documented task, code, metric definitions, and results, allowing other teams to run the same benchmark and compare against the published numbers; the evaluation contract and scoring are stable and reusable."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/tiago369/cross-engine-humanoid-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"public_reusable","publishers":[{"name":"Tiago369","organizationType":"community","sourceUrl":"https://github.com/tiago369/cross-engine-humanoid-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_company-mental-model-benchmark_e52dec26","familyId":"bmf_2a0a50cba0bc","name":"company-mental-model-benchmark","oneLine":"Research benchmark testing whether explicit company economic-model representations improve LLM's ability to identify compounding mechanisms. Compares direct synthesis, structured model, and structured model with critic on 8 companies.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Software & Cloud","Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/0xboyu/company-mental-model-benchmark","pdf":null,"project":null,"code":"https://github.com/0xboyu/company-mental-model-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"company-mental-model-benchmark Research benchmark for testing explicit company economic-model representations against direct LLM synthesis.","reasonCodes":["discovered via github","benchmark term in abstract","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:0xboyu/company-mental-model-benchmark"},"ranking":{"30d":{"score":34,"rank":103,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":326,"coverage":0.55,"confidence":"Low"}},"description":"Research benchmark testing whether explicit company economic-model representations improve LLM's ability to identify compounding mechanisms. Compares direct synthesis, structured model, and structured model with critic on 8 companies.","whyItMatters":"Addresses a research question about LLM reasoning, but the benchmark has no standalone comparison path and is designed to support a specific paper's finding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"27a775c7455d685e5f57bc7ea6a7bd095fc1cf193f0e80d58019ea47bcde6970"},"motivation":"company-mental-model-benchmark Research benchmark for testing explicit company economic-model representations against direct LLM synthesis.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"Defines a repeatable task suite with frozen evidence, three treatment comparisons, and a scoring rubric; repository provides runner and protocol for reuse."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/0xboyu/company-mental-model-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"public_reusable","displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_asr-model-benchmark_50d41822","familyId":"bmf_8795f3a05ca1","name":"asr-model-benchmark","oneLine":"Reproducible CPU benchmark comparing Whisper Small, Faster-Whisper Small, and Wav2Vec2 Base 960h under clean and 10 dB SNR noisy speech conditions. Evaluates Word Error Rate (WER), inference latency, real-time factor (RTF), memory usage, and robustness to controlled noise. Uses 20 clean and 20 noisy samples from the LibriSpeech ASR clean test set with fixed random seed.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/Anuragkokate09/asr-model-benchmark","pdf":null,"project":null,"code":"https://github.com/Anuragkokate09/asr-model-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"asr-model-benchmark Reproducible CPU benchmark comparing Whisper Small, Faster-Whisper Small, and Wav2Vec2 under clean and controlled noisy speech conditions.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:anuragkokate09/asr-model-benchmark"},"ranking":{"30d":{"score":34,"rank":104,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":327,"coverage":0.55,"confidence":"Low"}},"description":"Reproducible CPU benchmark comparing Whisper Small, Faster-Whisper Small, and Wav2Vec2 Base 960h under clean and 10 dB SNR noisy speech conditions. Evaluates Word Error Rate (WER), inference latency, real-time factor (RTF), memory usage, and robustness to controlled noise. Uses 20 clean and 20 noisy samples from the LibriSpeech ASR clean test set with fixed random seed.","whyItMatters":"Provides a controlled comparison of three widely used ASR models on CPU, measuring both accuracy and efficiency. Helps practitioners select models for resource-constrained, CPU-only deployments. The consistent protocol and noise injection methodology offer a baseline for evaluating other ASR models under similar conditions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"11dba55c29208501bd9279320efee1c25978b518dd6489d89c029c2246e1a4ad"},"motivation":"asr-model-benchmark Reproducible CPU benchmark comparing Whisper Small, Faster-Whisper Small, and Wav2Vec2 under clean and controlled noisy speech conditions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"Fixed evaluation protocol with defined metrics and dataset from LibriSpeech; code and results are available for reproducible comparison."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/Anuragkokate09/asr-model-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"score_submission","publishers":[{"name":"Anurag Kokate","organizationType":"community","sourceUrl":"https://github.com/Anuragkokate09/asr-model-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_agent-benchmark-2d-maze_4d511c00","familyId":"bmf_d82de6e84a4d","name":"agent-benchmark-2d-maze","oneLine":"Benchmark for evaluating interactive multimodal agents in structured 2D maze environments. Requires long-horizon interaction, partial observability, action execution, and recovery from mistakes. Includes maze specifications, environment implementations, model adapters, and a mechanism-aware progress scoring system. Provides scripts for running and reproducing evaluations.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Agents","Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/bluebunnyanon/agent-benchmark-2d-maze","pdf":null,"project":null,"code":"https://github.com/bluebunnyanon/agent-benchmark-2d-maze","data":null,"hfPaper":null},"evidence":{"snippet":"agent-benchmark-2d-maze # Agentic Evaluation in 2D Mazes This repository contains the code, benchmark instances, evaluation harness, and reproducibility artifacts accompanying an anonymous submission on evaluating interactive multimodal agents in structured 2D maze environments.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:bluebunnyanon/agent-benchmark-2d-maze"},"ranking":{"30d":{"score":34,"rank":105,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":328,"coverage":0.55,"confidence":"Low"}},"description":"Benchmark for evaluating interactive multimodal agents in structured 2D maze environments. Requires long-horizon interaction, partial observability, action execution, and recovery from mistakes. Includes maze specifications, environment implementations, model adapters, and a mechanism-aware progress scoring system. Provides scripts for running and reproducing evaluations.","whyItMatters":"Addresses the gap in evaluating agents on sequential, goal-directed tasks beyond single-step prediction. Offers a controlled environment with declarative task specifications and a transparent scoring mechanism, enabling comparison of different models' planning and interaction capabilities. The included human-playable interface and validation utilities support broader usability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"a2c5bd6118929ed81026a0906fefc6565780cfd9feb9fd7fdcd833f1d298d48f"},"motivation":"agent-benchmark-2d-maze # Agentic Evaluation in 2D Mazes This repository contains the code, benchmark instances, evaluation harness, and reproducibility artifacts accompanying an anonymous submission on evaluating interactive multimodal agents in structured 2D maze environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"Provides maze corpus, evaluation harness, scoring implementation, and reproduction instructions; task validation and solvability checks are included."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/bluebunnyanon/agent-benchmark-2d-maze","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T05:46:57.857309Z"},"evaluationMode":"public_reusable","publishers":[{"name":"Blue Bunny Anon","organizationType":"community","sourceUrl":"https://github.com/bluebunnyanon/agent-benchmark-2d-maze","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Agents","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_agent-app-benchmark_0bc5ce52","familyId":"bmf_407a33ad9273","name":"Agent App Benchmark","oneLine":"Public benchmark for GUI-based coding-agent applications, measuring application start, session switching, memory, and CPU performance using deterministic synthetic session corpora and trusted driver adapters.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Agents"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-08-22","firstSeenAt":"2026-08-24","recognitionConfidence":0.4,"links":{"report":"https://github.com/kyashrathore/agent-app-benchmark","pdf":null,"project":null,"code":"https://github.com/kyashrathore/agent-app-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"agent-app-benchmark Reproducible performance benchmarks for multi-harness coding-agent GUI applications # Agent App Benchmark Agent App Benchmark is a public, reproducible performance benchmark for multi-harness coding-agent GUI applications.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:kyashrathore/agent-app-benchmark"},"ranking":{"30d":{"score":34,"rank":106,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":329,"coverage":0.55,"confidence":"Low"}},"description":"Public benchmark for GUI-based coding-agent applications, measuring application start, session switching, memory, and CPU performance using deterministic synthetic session corpora and trusted driver adapters.","whyItMatters":"Provides a reproducible, standardized way to compare GUI-heavy agent applications, filling a gap in agent evaluation that often ignores UI performance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T08:57:07.998303Z","inputHash":"736ec78cf4fe3787d2f5ee686904984011d595b0dc51bd41f7b3dd1723d397d8"},"motivation":"agent-app-benchmark Reproducible performance benchmarks for multi-harness coding-agent GUI applications # Agent App Benchmark Agent App Benchmark is a public, reproducible performance benchmark for multi-harness coding-agent GUI applications.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:41:28.683304Z","model":"deepseek-v4-pro","decisionReason":"Public benchmark framework with canonical corpus and validation allows multiple applications to be scored under identical conditions."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/kyashrathore/agent-app-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:40:52.396154Z"},"evaluationMode":"score_submission","publishers":[{"name":"kyashrathore","organizationType":"community","sourceUrl":"https://github.com/kyashrathore/agent-app-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_recoveriq_46ac0f4c","familyId":"bmf_098c923d5b29","name":"recoveriq","oneLine":"RecoverIQ is an AI-assisted revenue recovery engine for failed payments with deterministic policy guardrails and a simulation-based evaluation harness. It includes a synthetic dataset of 10,000 payment transactions and a repeatable benchmark comparing baseline, ML, and agentic policies on recovery rate and safety.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Software & Cloud","Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-21","firstSeenAt":"2026-08-22","recognitionConfidence":0.4,"links":{"report":"https://github.com/aryanthemscoder/recoveriq","pdf":null,"project":"http://127.0.0.1:8000`","code":"https://github.com/aryanthemscoder/recoveriq","data":null,"hfPaper":null},"evidence":{"snippet":"recoveriq AI-assisted revenue recovery engine for failed payments with deterministic policy guardrails, simulation, and benchmark-driven evaluation.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:aryanthemscoder/recoveriq"},"ranking":{"30d":{"score":34,"rank":107,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":330,"coverage":0.55,"confidence":"Low"}},"description":"RecoverIQ is an AI-assisted revenue recovery engine for failed payments with deterministic policy guardrails and a simulation-based evaluation harness. It includes a synthetic dataset of 10,000 payment transactions and a repeatable benchmark comparing baseline, ML, and agentic policies on recovery rate and safety.","whyItMatters":"It provides a framework for evaluating AI-assisted payment recovery policies under controlled conditions, focusing on safety constraints and measurable outcomes. The benchmark could inform decisions on deploying such systems in fintech operations.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"ee2f66fc4895d7a992bd0db369bbdac3b6181554e83620687aadd765ed029e24"},"motivation":"recoveriq AI-assisted revenue recovery engine for failed payments with deterministic policy guardrails, simulation, and benchmark-driven evaluation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"Repeatable benchmark script compares baseline, ML, and agentic policies with deterministic scoring; code and run instructions are public."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/aryanthemscoder/recoveriq","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-22T19:19:35.289219Z"},"evaluationMode":"score_submission","displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_qwen3-8-27b-inference-benchmark-4090_328340c0","familyId":"bmf_c8f88fa92b7e","name":"qwen3.8-27b-inference-benchmark-4090","oneLine":"Benchmark evaluating inference throughput and accuracy of four serving configurations for Qwen3.8-27B on RTX 4090. Includes performance metrics per request and accuracy scores on three suites, with protocol and metadata.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Inspectable","releasedAt":"2026-08-21","firstSeenAt":"2026-08-22","recognitionConfidence":0.4,"links":{"report":"https://huggingface.co/datasets/pxzleo/qwen3.8-27b-inference-benchmark-4090","pdf":null,"project":null,"code":null,"data":"https://huggingface.co/datasets/pxzleo/qwen3.8-27b-inference-benchmark-4090","hfPaper":null},"evidence":{"snippet":"Qwen3.8-27B Inference Benchmark on RTX 4090 48GB 中文说明 · GitHub benchmark repository Structured performance and accuracy results for four real Qwen3.8-27B serving configurations on an NVIDIA RTX 4090 48 GB workstation.","reasonCodes":["discovered via huggingface","benchmark term in abstract","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":53,"hfDatasetLikes":1},"source":{"type":"huggingface","id":"huggingface:pxzleo/qwen3.8-27b-inference-benchmark-4090"},"ranking":{"30d":{"score":46,"rank":87,"coverage":0.15,"confidence":"Low","datasetDownloadRank":16,"datasetRankPopulation":20},"90d":{"score":40,"rank":265,"coverage":0.3,"confidence":"Low","datasetDownloadRank":43,"datasetRankPopulation":51}},"description":"Benchmark evaluating inference throughput and accuracy of four serving configurations for Qwen3.8-27B on RTX 4090. Includes performance metrics per request and accuracy scores on three suites, with protocol and metadata.","whyItMatters":"Provides reproducible comparison of local LLM serving options on consumer hardware, covering latency, throughput, and accuracy trade-offs relevant for deployment decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T03:59:08.558907Z","inputHash":"14da0814b700d198d488d8ad465ae4c8116969cee5abcbd0dbd801e1116e2685"},"motivation":"Qwen3.8-27B Inference Benchmark on RTX 4090 48GB 中文说明 · GitHub benchmark repository Structured performance and accuracy results for four real Qwen3.8-27B serving configurations on an NVIDIA RTX 4090 48 GB workstation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-22T19:19:45.406000Z","model":"deepseek-v4-flash","decisionReason":"Defines a clear benchmark protocol, publishes per-request data, and provides a public dataset for reuse."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://huggingface.co/datasets/pxzleo/qwen3.8-27b-inference-benchmark-4090","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-22T19:19:35.289219Z"},"evaluationMode":"public_reusable","publishers":[{"name":"pxzleo","organizationType":"community","sourceUrl":"https://huggingface.co/datasets/pxzleo/qwen3.8-27b-inference-benchmark-4090","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_llm-summarization-benchmark-ptbr_c5bebc73","familyId":"bmf_d1dd765edb10","name":"llm-summarization-benchmark-ptbr","oneLine":"Benchmark for reference-faithful summarization in Brazilian Portuguese across nine local LLMs, with six runs per model and a pre-registered rubric scoring fidelity, coverage, format, and fluency.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-21","firstSeenAt":"2026-08-22","recognitionConfidence":0.4,"links":{"report":"https://github.com/gbbarra/llm-summarization-benchmark-ptbr","pdf":null,"project":"https://doi.org/10.5281/zenodo.22042502","code":"https://github.com/gbbarra/llm-summarization-benchmark-ptbr","data":"https://zenodo.org/badge/DOI/10.5281/zenodo.22042502.svg","hfPaper":null},"evidence":{"snippet":"llm-summarization-benchmark-ptbr # LLM Summarization Benchmark — Brazilian Portuguese [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22042502.svg)](https://doi.org/10.5281/zenodo.22042502) *Six runs per model, a pre-registered rubric, and a trap-laden source text: which local LLMs summarize a technical report with strict faithfulness to the reference — and how fast — on a mini PC with no discrete GPU?* A fully reproducible benchmark of **reference-faithful summarization** across 9 local LLM","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:gbbarra/llm-summarization-benchmark-ptbr"},"ranking":{"30d":{"score":34,"rank":108,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":331,"coverage":0.55,"confidence":"Low"}},"description":"Benchmark for reference-faithful summarization in Brazilian Portuguese across nine local LLMs, with six runs per model and a pre-registered rubric scoring fidelity, coverage, format, and fluency.","whyItMatters":"Addresses scarcity of Portuguese summarization benchmarks and provides reproducible protocol with complete outputs and analysis, enabling comparison of local models on a mini PC.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T03:59:08.558907Z","inputHash":"642b6ac80d3de0c49e13e5cac168184fa7c11a39a043c32948ffa3a3e99ee3ce"},"motivation":"llm-summarization-benchmark-ptbr # LLM Summarization Benchmark — Brazilian Portuguese [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22042502.svg)](https://doi.org/10.5281/zenodo.22042502) *Six runs per model, a pre-registered rubric, and a trap-laden source text: which local LLMs summarize a technical report with strict faithfulness to the reference — and how fast — on a mini PC with no discrete GPU?* A fully…","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-22T19:19:45.406000Z","model":"deepseek-v4-flash","decisionReason":"Provides a complete, reproducible protocol, published raw data and results, and a DOI for citation."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/gbbarra/llm-summarization-benchmark-ptbr","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-22T19:19:35.289219Z"},"evaluationMode":"public_reusable","publishers":[{"name":"gbbarra","organizationType":"community","sourceUrl":"https://github.com/gbbarra/llm-summarization-benchmark-ptbr","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_llm-free-benchmark_d7c7d60c","familyId":"bmf_aa59460ae398","name":"llm-free-benchmark","oneLine":"Dependency-free benchmark for free-tier LLMs on OpenRouter. Runs a fixed battery of seven typed tasks (logic, brevity, Russian, reasoning, code_count, code_regex, code_class) and grades each answer programmatically on a 0-5 scale. Produces a ranked report with per-model average, total, success rate, and latency.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-21","firstSeenAt":"2026-08-22","recognitionConfidence":0.4,"links":{"report":"https://github.com/Miolonixc/llm-free-benchmark","pdf":null,"project":"https://openrouter.ai/keys","code":"https://github.com/Miolonixc/llm-free-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"llm-free-benchmark Dependency-free benchmark for free-tier LLMs on OpenRouter.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:miolonixc/llm-free-benchmark"},"ranking":{"30d":{"score":34,"rank":109,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":332,"coverage":0.55,"confidence":"Low"}},"description":"Dependency-free benchmark for free-tier LLMs on OpenRouter. Runs a fixed battery of seven typed tasks (logic, brevity, Russian, reasoning, code_count, code_regex, code_class) and grades each answer programmatically on a 0-5 scale. Produces a ranked report with per-model average, total, success rate, and latency.","whyItMatters":"Provides a lightweight, reproducible method to compare free-tier LLMs on a diverse set of tasks, helping users choose cost-effective models. The sequential request design with cooldowns mitigates rate-limit artifacts, making scores more reflective of model quality.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:41:44.885071Z","inputHash":"f08ef6329d3d20b1c8a142a5652f6f6ab022360cc51c46871de63e51cb685ccc"},"motivation":"llm-free-benchmark Dependency-free benchmark for free-tier LLMs on OpenRouter.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-24T07:05:30.194517Z","model":"deepseek-v4-pro","decisionReason":"Scripts, task definitions, and grading scheme are provided; results are published and can be regenerated with an API key."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/Miolonixc/llm-free-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-22T19:19:35.289219Z"},"evaluationMode":"score_submission","publishers":[{"name":"Miolonixc","organizationType":"community","sourceUrl":"https://github.com/Miolonixc/llm-free-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_financial-sentiment-benchmark_ea047ce6","familyId":"bmf_1a3121aeb0c6","name":"financial-sentiment-benchmark","oneLine":"Benchmark comparing four approaches to financial-news sentiment classification on a fixed 300-sentence subset, measuring macro-F1, latency, and cost across fine-tuned and prompted models.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-21","firstSeenAt":"2026-08-22","recognitionConfidence":0.4,"links":{"report":"https://github.com/gauthamRohan/financial-sentiment-benchmark","pdf":null,"project":null,"code":"https://github.com/gauthamRohan/financial-sentiment-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"financial-sentiment-benchmark Fine-tune a small model or prompt a big one?","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:gauthamrohan/financial-sentiment-benchmark"},"ranking":{"30d":{"score":34,"rank":110,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":333,"coverage":0.55,"confidence":"Low"}},"description":"Benchmark comparing four approaches to financial-news sentiment classification on a fixed 300-sentence subset, measuring macro-F1, latency, and cost across fine-tuned and prompted models.","whyItMatters":"Provides reproducible comparison of fine-tuning versus prompting for a structured classification task, with implications for cost and speed in practical applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T03:59:08.558907Z","inputHash":"07e0cd434a2681e9573176929d64c8accb1761b4546637e20ae211400e0332c6"},"motivation":"financial-sentiment-benchmark Fine-tune a small model or prompt a big one?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-22T19:19:45.406000Z","model":"deepseek-v4-flash","decisionReason":"Provides a reproducible classification task with a fixed dataset, clear evaluation metrics (macro-F1), and public code for running the benchmark, making it reusable by other teams."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/gauthamRohan/financial-sentiment-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-22T19:19:35.289219Z"},"evaluationMode":"public_reusable","displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_annomi-counselling-dialogue-analysis_beb022e0","familyId":"bmf_22112cbfe0af","name":"annomi-counselling-dialogue-analysis","oneLine":"Project evaluating therapist-behaviour classification on AnnoMI corpus with transcript-grouped evaluation, comparing baseline and RoBERTa models. Includes secondary experiments on summarisation and forecasting.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences","Finance & Economics"],"primaryDomain":"Health & Life Sciences","industrySectors":["Software & Cloud","Pharma & Biotech","Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-08-21","firstSeenAt":"2026-08-22","recognitionConfidence":0.4,"links":{"report":"https://github.com/abdullahuseyinli-dot/annomi-counselling-dialogue-analysis","pdf":null,"project":null,"code":"https://github.com/abdullahuseyinli-dot/annomi-counselling-dialogue-analysis","data":null,"hfPaper":null},"evidence":{"snippet":"annomi-counselling-dialogue-analysis Transcript-grouped NLP benchmark for therapist-behaviour classification on AnnoMI counselling machine-learning natural-language-processing reproducibility roberta text-classification # AnnoMI Counselling Dialogue Analysis A reproducible NLP benchmark for therapist-behaviour classification on the [AnnoMI](https://github.com/uccollab/AnnoMI) counselling-dialogue corpus.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:abdullahuseyinli-dot/annomi-counselling-dialogue-analysis"},"ranking":{"30d":{"score":34,"rank":111,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":334,"coverage":0.55,"confidence":"Low"}},"description":"Project evaluating therapist-behaviour classification on AnnoMI corpus with transcript-grouped evaluation, comparing baseline and RoBERTa models. Includes secondary experiments on summarisation and forecasting.","whyItMatters":"Potential to provide a reproducible benchmark for counselling dialogue analysis, but evidence is incomplete as of review.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T03:59:08.558907Z","inputHash":"787ca2c298b51a614fec00b9c190222852abc2c2b7b5a7c51adcab49fe4baff1"},"motivation":"annomi-counselling-dialogue-analysis Transcript-grouped NLP benchmark for therapist-behaviour classification on AnnoMI counselling machine-learning natural-language-processing reproducibility roberta text-classification # AnnoMI Counselling Dialogue Analysis A reproducible NLP benchmark for therapist-behaviour classification on the [AnnoMI](https://github.com/uccollab/AnnoMI) counselling-dialogue corpus.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-22T19:19:45.406000Z","model":"deepseek-v4-flash","decisionReason":"Presents a fixed dataset, clear evaluation design with grouped splits, statistical testing, and public code for replication, making it a reusable benchmark for other teams."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/abdullahuseyinli-dot/annomi-counselling-dialogue-analysis","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-22T19:19:35.289219Z"},"evaluationMode":"public_reusable","displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"cross-domain"},{"id":"bm_ai-sim-benchmark_0f263e79","familyId":"bmf_6432f57efebd","name":"ai-sim-benchmark","oneLine":"Reproducible coding challenge comparing autonomous AI coding agents on implementing a 3D water simulation, measuring capability ceiling with no time or budget cap. Includes pilot runs and protocol.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Geometric reasoning"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-21","firstSeenAt":"2026-08-22","recognitionConfidence":0.4,"links":{"report":"https://github.com/AiondaDotCom/ai-sim-benchmark","pdf":null,"project":null,"code":"https://github.com/AiondaDotCom/ai-sim-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"ai-sim-benchmark A reproducible 3D water-simulation challenge for comparing autonomous AI coding agents # AI Coding Agent Water Simulation Benchmark This repository contains a reproducible coding challenge for comparing autonomous coding agents such as Claude Code, Kimi Code, OpenCode, and similar tools.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:aiondadotcom/ai-sim-benchmark"},"ranking":{"30d":{"score":34,"rank":112,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":335,"coverage":0.55,"confidence":"Low"}},"description":"Reproducible coding challenge comparing autonomous AI coding agents on implementing a 3D water simulation, measuring capability ceiling with no time or budget cap. Includes pilot runs and protocol.","whyItMatters":"Addresses evaluation of AI coding agents in a self-contained complex task, but current state shows evolving protocol and incomplete runs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T03:59:08.558907Z","inputHash":"ea5ceb4a3003b40f972903fe478fe8d090f6aae0cd65057cf9fb3d1e920df7e5"},"motivation":"ai-sim-benchmark A reproducible 3D water-simulation challenge for comparing autonomous AI coding agents # AI Coding Agent Water Simulation Benchmark This repository contains a reproducible coding challenge for comparing autonomous coding agents such as Claude Code, Kimi Code, OpenCode, and similar tools.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-22T19:19:45.406000Z","model":"deepseek-v4-flash","decisionReason":"Offers a public, reproducible challenge with a fixed prompt, clear success criteria, and public infrastructure for running the benchmark, making it reusable for model comparison."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/AiondaDotCom/ai-sim-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-22T19:19:35.289219Z"},"evaluationMode":"public_reusable","displayEligible":false,"capabilityGroups":["Agents","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_agent-model-bench_4106c088","familyId":"bmf_edbdc48fb887","name":"agent-model-bench","oneLine":"Benchmark evaluating tool-calling and strict JSON output capabilities across models, with latency and cost metrics. Fixed 32-case suite with exact scoring and provider pluggability.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-21","firstSeenAt":"2026-08-22","recognitionConfidence":0.4,"links":{"report":"https://github.com/ajbermudezh22/agent-model-bench","pdf":null,"project":"https://ajbermudezh22.github.io/agent-model-bench/","code":"https://github.com/ajbermudezh22/agent-model-bench","data":null,"hfPaper":null},"evidence":{"snippet":"agent-model-bench Which model should power your agent?","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:ajbermudezh22/agent-model-bench"},"ranking":{"30d":{"score":34,"rank":113,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":336,"coverage":0.55,"confidence":"Low"}},"description":"Benchmark evaluating tool-calling and strict JSON output capabilities across models, with latency and cost metrics. Fixed 32-case suite with exact scoring and provider pluggability.","whyItMatters":"Provides practical comparison for agent framework selection, focusing on functional correctness and formatting discipline that affect production integration.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T03:59:08.558907Z","inputHash":"3d25abacb324b45c23e74539a926095c2c6a8713554188fd81c605f89035e0c9"},"motivation":"agent-model-bench Which model should power your agent?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-22T19:19:45.406000Z","model":"deepseek-v4-flash","decisionReason":"Provides a public, reproducible set of test cases with clear scoring and code for running the benchmark, making it a reusable evaluation tool for model comparison."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/ajbermudezh22/agent-model-bench","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-22T19:19:35.289219Z"},"evaluationMode":"public_reusable","publishers":[{"name":"ajbermudezh22","organizationType":"community","sourceUrl":"https://github.com/ajbermudezh22/agent-model-bench","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_vgi-bench_4a5d68ac","familyId":"bmf_756186ceeff4","name":"VGI-BENCH","oneLine":"VGI-bench evaluates visual reasoning in video generation models across 27 tasks and 810 instances, organized by task domains and skill tags.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-20","firstSeenAt":"2026-08-21","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.19583","pdf":"https://arxiv.org/pdf/2608.19583","project":null,"code":null,"data":null,"hfPaper":null},"evidence":{"snippet":"To this end, we introduce VGI-bench, containing 27 tasks and 810 instances, organized by a two-level taxonomy of task domains and skill tags for fine-grained evaluation of visual reasoning capabilities of video generation models.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.19583"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"VGI-bench evaluates visual reasoning in video generation models across 27 tasks and 810 instances, organized by task domains and skill tags.","whyItMatters":"Addresses the need for calibrated evaluation of emerging visual reasoning capabilities in video generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:12.448463Z","inputHash":"7834de0eed31c48ff6773dae1c9a88e393575aebb22ddec7505f05b4cef9e5d0"},"motivation":"Recent studies suggest that video generation models can exhibit certain forms of zero-shot visual reasoning through generated frames.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-21T04:30:12.448463Z","model":"deepseek-v4-flash","decisionReason":"Offers a fixed dataset with clear evaluation criteria and public release plans."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.19583","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-21T04:30:09.569171Z"},"evaluationMode":"public_reusable","displayEligible":true,"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_the-value-engine-benchmark_3b9623cf","familyId":"bmf_7c1e90c7b46d","name":"the-value-engine-benchmark","oneLine":"VEB is an evidence-graded RL environment and benchmark for evaluating LLM agents on multi-touch enterprise-sales negotiation. Agents act as B2B sellers in multi-turn negotiations against a stochastic LLM buyer, graded with a sales-methodology rubric producing scalar rewards and structured diagnostic grades.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/rudycelekli/the-value-engine-benchmark","pdf":null,"project":"https://thevalueengine.ai","code":"https://github.com/rudycelekli/the-value-engine-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"the-value-engine-benchmark VEB: an evidence-graded RL environment and benchmark for LLM agents on multi-touch enterprise-sales negotiation.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:rudycelekli/the-value-engine-benchmark"},"ranking":{"30d":{"score":34,"rank":114,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":337,"coverage":0.55,"confidence":"Low"}},"description":"VEB is an evidence-graded RL environment and benchmark for evaluating LLM agents on multi-touch enterprise-sales negotiation. Agents act as B2B sellers in multi-turn negotiations against a stochastic LLM buyer, graded with a sales-methodology rubric producing scalar rewards and structured diagnostic grades.","whyItMatters":"VEB addresses the need for evaluating long-horizon planning, value framing, price discipline, and honesty under pressure in sales negotiations, which single-turn benchmarks do not stress. It provides a methodology-controlled environment for training and evaluating agents, with a focus on reproducible evaluation and a leaderboard for model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T09:02:46.503495Z","inputHash":"9ce4a58ce6c21da969ea09f8060167d55578ad77f6b300e8b724549347ff929d"},"motivation":"the-value-engine-benchmark VEB: an evidence-graded RL environment and benchmark for LLM agents on multi-touch enterprise-sales negotiation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T06:56:13.762703Z","model":"deepseek-v4-pro","decisionReason":"Provides a runnable environment, fixed task set, explicit scoring rubric, committed validation procedures, and a public repository for third-party runs and submissions."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/rudycelekli/the-value-engine-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"score_submission","publishers":[{"name":"The Value Engine","organizationType":"company-research-lab","sourceUrl":"https://thevalueengine.ai","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_swe-bench_536d758e","familyId":"bmf_c0124d8dc037","name":"SWE-bench Science","oneLine":"SWE-bench Science evaluates coding agents on 119 repository-level tasks from 98 GitHub repositories spanning 20 scientific domains, organized into Issue-driven, Expert-exploratory, and Engineering-integration paradigms. It measures pass@1 for repairing scientific software.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Code"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-20","firstSeenAt":"2026-08-21","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.19799","pdf":"https://arxiv.org/pdf/2608.19799","project":null,"code":null,"data":null,"hfPaper":null},"evidence":{"snippet":"We introduce \\textbf{SWE-bench Science}, a repository-level benchmark for scientific software engineering comprising 119 tasks from 98 GitHub repositories across 20 scientific domains.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":61,"hfDailySubmittedAt":"2026-08-21T00:00:00.000Z","githubStars":52,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.19799"},"ranking":{"30d":{"score":89,"rank":5,"coverage":0.85,"confidence":"High"},"90d":{"score":78,"rank":27,"coverage":0.7,"confidence":"Medium"}},"description":"SWE-bench Science evaluates coding agents on 119 repository-level tasks from 98 GitHub repositories spanning 20 scientific domains, organized into Issue-driven, Expert-exploratory, and Engineering-integration paradigms. It measures pass@1 for repairing scientific software.","whyItMatters":"Existing coding agent benchmarks emphasize aggregate success and provide limited insight into failures in scientific software. SWE-bench Science provides a broad testbed for studying capabilities and failure mechanisms, noting that even top agents achieve pass@1 below 50%, helping developers identify where scientific knowledge helps or hinders repair.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T05:19:04.530412Z","inputHash":"270926c4f9204b4d25d8c0f9ecee3928e690eed0c86a223c8c2082cecd5cd884"},"motivation":"Software increasingly functions as part of the scientific instrument itself, making failures in scientific code capable of compromising not only program behavior but also the evidence underlying scientific conclusions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-21T04:30:12.448463Z","model":"deepseek-v4-flash","decisionReason":"Presents a fixed dataset with clear task structure and public release."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.19799","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-21T04:30:09.569171Z"},"evaluationMode":"public_reusable","displayEligible":true,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_silentbug-bench_29b06a59","familyId":"bmf_9af0df30b045","name":"silentbug-bench","oneLine":"silentbug-bench evaluates LLM agents on detecting silent defects in ML training code. Agents receive a small training repository with an injected defect and must identify the defect from a fixed taxonomy, localize it, and provide a patch. Scoring is automated via a program that applies the patch, retrains, and evaluates six components.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/Json604/silentbug-bench","pdf":null,"project":null,"code":"https://github.com/Json604/silentbug-bench","data":null,"hfPaper":null},"evidence":{"snippet":"silentbug-bench ML training defects that never crash.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:json604/silentbug-bench"},"ranking":{"30d":{"score":34,"rank":115,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":338,"coverage":0.55,"confidence":"Low"}},"description":"silentbug-bench evaluates LLM agents on detecting silent defects in ML training code. Agents receive a small training repository with an injected defect and must identify the defect from a fixed taxonomy, localize it, and provide a patch. Scoring is automated via a program that applies the patch, retrains, and evaluates six components.","whyItMatters":"This benchmark addresses the challenge of detecting training defects that do not crash or obviously degrade performance, which are difficult to identify manually. It provides a standardized evaluation for agent performance on code understanding, defect localization, and patch correctness, with clear scoring and baselines.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T09:02:46.503495Z","inputHash":"db183fcf1eb1847c3aff7344740324a46e521b311a5df2c8d338a3be96979de3"},"motivation":"silentbug-bench ML training defects that never crash.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T06:56:13.762703Z","model":"deepseek-v4-pro","decisionReason":"The repository includes a locked task set, deterministic scoring components, baselines, and a harness that supports third-party agent submissions, enabling comparable evaluations."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/Json604/silentbug-bench","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"score_submission","publishers":[{"name":"Json604","organizationType":"community","sourceUrl":"https://github.com/Json604/silentbug-bench","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_silent-misresolution-benchmark_79846646","familyId":"bmf_39f93253928c","name":"silent-misresolution-benchmark","oneLine":"This repository provides a benchmark and scorer for detecting silent misresolution in dictation cleanup, where a system deletes a spoken revision but produces grammatically correct output that changes the meaning. It includes a corpus of revision sites and a scoring methodology distinguishing silent from visible errors.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Software & Cloud","Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/hemachandra666/silent-misresolution-benchmark","pdf":null,"project":null,"code":"https://github.com/hemachandra666/silent-misresolution-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"silent-misresolution-benchmark A benchmark for silent misresolution in dictation cleanup: when a system deletes what you said and the output still reads perfectly.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:hemachandra666/silent-misresolution-benchmark"},"ranking":{"30d":{"score":34,"rank":116,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":339,"coverage":0.55,"confidence":"Low"}},"description":"This repository provides a benchmark and scorer for detecting silent misresolution in dictation cleanup, where a system deletes a spoken revision but produces grammatically correct output that changes the meaning. It includes a corpus of revision sites and a scoring methodology distinguishing silent from visible errors.","whyItMatters":"This benchmark exposes a failure mode in dictation systems that existing metrics like edit rate and WER cannot detect. It quantifies the prevalence of silent misresolution and highlights limitations of current evaluation approaches, which could inform better quality metrics for speech-to-text products.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T09:02:46.503495Z","inputHash":"7a82e16ed47864e083f32642b65247bb248b01389f2b52e573dfadff28237b64"},"motivation":"silent-misresolution-benchmark A benchmark for silent misresolution in dictation cleanup: when a system deletes what you said and the output still reads perfectly.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T06:56:13.762703Z","model":"deepseek-v4-pro","decisionReason":"The repository includes a benchmark and scorer with a defined scoring contract and public reuse path, despite small corpus and methodological limitations. It is intended for measuring a specific error type and can be used by other teams."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/hemachandra666/silent-misresolution-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"public_reusable","displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_sbmllm-bench_68cf829e","familyId":"bmf_33e4d26b1444","name":"SBMLLM-Bench","oneLine":"SBMLLM-Bench is a manually curated dataset and benchmark for evaluating LLMs on reconstructing executable systems-biology models from scientific papers. Models convert publications into Antimony models, and performance is measured by simulation success, species recovery, reaction accuracy, and dynamic reproducibility.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/cosbi-research/SBMLLM-Bench","pdf":null,"project":"https://www.cosbi.eu/contact","code":"https://github.com/cosbi-research/SBMLLM-Bench","data":null,"hfPaper":null},"evidence":{"snippet":"SBMLLM-Bench Benchmark LLMs on their ability to reconstruct executable systems-biology models from scientific papers.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:cosbi-research/sbmllm-bench"},"ranking":{"30d":{"score":34,"rank":117,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":340,"coverage":0.55,"confidence":"Low"}},"description":"SBMLLM-Bench is a manually curated dataset and benchmark for evaluating LLMs on reconstructing executable systems-biology models from scientific papers. Models convert publications into Antimony models, and performance is measured by simulation success, species recovery, reaction accuracy, and dynamic reproducibility.","whyItMatters":"This benchmark addresses the need for automated model reconstruction in systems biology, which is time-consuming and requires expertise. It provides a standardized evaluation for LLM-based model generation and repair, with clear metrics for structural and dynamic fidelity, facilitating comparison of approaches.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T09:02:46.503495Z","inputHash":"b973248419d89013807d1c3fc497b8a8e97cdf542ba8f7ba22307340ee7aee60"},"motivation":"SBMLLM-Bench Benchmark LLMs on their ability to reconstruct executable systems-biology models from scientific papers.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T06:56:13.762703Z","model":"deepseek-v4-pro","decisionReason":"The repository contains the dataset, reference models, scoring metrics, and a reproducible Snakemake workflow, making it usable by other teams. The task and scoring are clearly defined."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/cosbi-research/SBMLLM-Bench","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"public_reusable","publishers":[{"name":"COSBI","organizationType":"academic-lab","sourceUrl":"https://www.cosbi.eu/contact","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_rule-compliant-visual-spatial-planning-for_2793101a","familyId":"bmf_1b6b5724735c","name":"Rule-Compliant Visual Spatial Planning for Multimodal Large Language Models","oneLine":"RuleMaze evaluates multimodal large language models on rule-compliant visual spatial planning. Models navigate visual mazes with natural-language rule constraints, requiring joint perception, rule interpretation, and action planning. The benchmark provides procedural maze generation, rule-conditioned trajectories, and seen/unseen rule splits with validators.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning"],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-21","recognitionConfidence":0.55,"links":{"report":"https://arxiv.org/abs/2608.20237","pdf":"https://arxiv.org/pdf/2608.20237","project":null,"code":"https://github.com/oceanflowlab/RuleMaze","data":null,"hfPaper":null},"evidence":{"snippet":"To address this gap, we introduce RuleMaze, a controllable benchmark in which MLLMs must navigate mazes while obeying natural-language rules of varying complexity.","reasonCodes":["benchmark term in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.20237"},"ranking":{"30d":{"score":50,"rank":67,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":276,"coverage":0.55,"confidence":"Low"}},"description":"RuleMaze evaluates multimodal large language models on rule-compliant visual spatial planning. Models navigate visual mazes with natural-language rule constraints, requiring joint perception, rule interpretation, and action planning. The benchmark provides procedural maze generation, rule-conditioned trajectories, and seen/unseen rule splits with validators.","whyItMatters":"RuleMaze addresses the lack of benchmarks for evaluating MLLMs on grounded spatial planning under explicit rule constraints. It provides a controlled environment to measure rule compliance and generalization, offering practical value for testing and developing models that must follow complex instructions in visual environments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T14:37:24.181170Z","inputHash":"00b4dfbcfe1852259a3dbe5aad72a34901c9066d190c182bb7b1cf92333ba30d"},"motivation":"Multimodal large language models (MLLMs) combine linguistic reasoning with visual perception, yet their ability to perform visual spatial planning under explicit or previously unseen rule constraints remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-21T04:30:12.448463Z","model":"deepseek-v4-flash","decisionReason":"Provides a public code repository and dataset for systematic evaluation."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.20237","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-21T04:30:09.569171Z"},"evaluationMode":"public_reusable","publishers":[{"name":"OceanFlow Lab","organizationType":"academic-lab","sourceUrl":"https://github.com/oceanflowlab/RuleMaze","role":"benchmark-publisher"}],"displayEligible":true,"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_rtcbench_9cd327fe","familyId":"bmf_dd843208c5ec","name":"rtcbench","oneLine":"RTCbench evaluates LLMs on commissioning closed-loop control systems. A submission is a controller that is wired to a dynamic process simulation and judged on plant performance. It includes ten plants, safety gates, and an anchored scoring baseline.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/aiast1/rtcbench","pdf":null,"project":null,"code":"https://github.com/aiast1/rtcbench","data":null,"hfPaper":null},"evidence":{"snippet":"rtcbench A benchmark for LLMs that commission real closed-loop control systems.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:aiast1/rtcbench"},"ranking":{"30d":{"score":34,"rank":118,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":341,"coverage":0.55,"confidence":"Low"}},"description":"RTCbench evaluates LLMs on commissioning closed-loop control systems. A submission is a controller that is wired to a dynamic process simulation and judged on plant performance. It includes ten plants, safety gates, and an anchored scoring baseline.","whyItMatters":"RTCbench addresses the gap in evaluating LLMs for control engineering tasks beyond theoretical quizzes, focusing on real closed-loop performance. It provides a reproducible benchmark with validated tasks, safety constraints, and variance-aware ranking, useful for comparing models on control commissioning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T09:02:46.503495Z","inputHash":"8c8616952124927ff87a59d61fd3b7b94dfd2c9b2ab11128b58674e500b0f135"},"motivation":"rtcbench A benchmark for LLMs that commission real closed-loop control systems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T06:56:13.762703Z","model":"deepseek-v4-pro","decisionReason":"The repository provides a scoring harness, task definitions, reference controllers, and a leaderboard, with a clear interface for submitting controllers. The scoring contract is stable and publicly reusable."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/aiast1/rtcbench","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"score_submission","publishers":[{"name":"aiast1","organizationType":"community","sourceUrl":"https://github.com/aiast1/rtcbench","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_pl-nba_96aca2fb","familyId":"bmf_945d8d97150d","name":"PL-NBA","oneLine":"PL-NBA is a possession-level basketball video dataset with 11,000 clips and 31,567 annotated events supporting multiple visual understanding tasks.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-20","firstSeenAt":"2026-08-21","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.19646","pdf":"https://arxiv.org/pdf/2608.19646","project":null,"code":null,"data":null,"hfPaper":null},"evidence":{"snippet":"Experimental results show that existing methods achieve limited performance on above four tasks, demonstrating that PL-NBA is a challenging benchmark for sports video understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.19646"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PL-NBA is a possession-level basketball video dataset with 11,000 clips and 31,567 annotated events supporting multiple visual understanding tasks.","whyItMatters":"Preserves temporal continuity for tasks like action anticipation and tactic analysis in sports video understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:12.448463Z","inputHash":"f332eb5cf67bd9c059d77693c8a2b1bbd2ef6b2feaf6cb976d57e93dd05fdbc3"},"motivation":"Visual understanding in sports has emerged as a hot topic in computer vision in recent years.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-21T04:30:12.448463Z","model":"deepseek-v4-flash","decisionReason":"Provides a fixed dataset with annotations and supports multiple tasks."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.19646","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-21T04:30:09.569171Z"},"evaluationMode":"public_reusable","displayEligible":true,"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_personalbench_05e1f48e","familyId":"bmf_83ab15035e3e","name":"PersonalBench","oneLine":"PersonalBench evaluates inference-time LLM personalization by measuring authorship resemblance across 50 authors and 1,000 generations, using LUAR, LLM-as-judge, and stylometrics. It provides metrics like LUAR similarity and AUC.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-20","firstSeenAt":"2026-08-21","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.19746","pdf":"https://arxiv.org/pdf/2608.19746","project":null,"code":null,"data":null,"hfPaper":null},"evidence":{"snippet":"We introduce PersonalBench, a benchmark that evaluates inference-time personalization methods through three independent lenses: LUAR (a trained authorship verification model), an LLM-as-judge, and automated stylometrics.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.19746"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PersonalBench evaluates inference-time LLM personalization by measuring authorship resemblance across 50 authors and 1,000 generations, using LUAR, LLM-as-judge, and stylometrics. It provides metrics like LUAR similarity and AUC.","whyItMatters":"Fills gap in personalization evaluation by focusing on authorship resemblance rather than task accuracy. PersonalBench serves as a calibrated measuring stick, showing current methods modulate style but fail to bridge the human-LLM authorship gap, informing future personalization approaches.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T05:19:04.530412Z","inputHash":"13d0ad09a7ce0eb1a4846c3e26d55e9906c66766379d36a1d64efa5832c0274e"},"motivation":"Personalized text generation aims to make LLMs write in a specific individual's style, yet existing benchmarks measure task accuracy or preference alignment rather than whether the model's output actually resembles the target author's writing.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-21T04:30:12.448463Z","model":"deepseek-v4-flash","decisionReason":"Offers a fixed dataset and protocol for evaluating personalization methods."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.19746","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-21T04:30:09.569171Z"},"evaluationMode":"public_reusable","displayEligible":true,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_openbenchmark_199b1148","familyId":"bmf_9da9d25e3255","name":"OpenBenchmark","oneLine":"OpenBenchmark is a platform for turning agent trajectories into custom benchmarks and ranking models on score, cost, and speed. It allows importing traces, drafting tasks with an AI agent, and running evaluations across models.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/crispyberry/OpenBenchmark","pdf":null,"project":"https://pi.dev","code":"https://github.com/crispyberry/OpenBenchmark","data":null,"hfPaper":null},"evidence":{"snippet":"ai-agents benchmark evaluation llm llm-evaluation nextjs observability openrouter postgres typescript # OpenBenchmark **English** · [简体中文](README.zh-CN.md) · [日本語](README.ja.md) · [Français](README.fr.md) · [Español](README.es.md) **Turn real agent trajectories into runnable benchmarks, then rank models, harnesses, and your own agents on score, cost, and speed.** Standard benchmarks tell you which model is good at someone else's problem.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:crispyberry/openbenchmark"},"ranking":{"30d":{"score":50,"rank":68,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":277,"coverage":0.55,"confidence":"Low"}},"description":"OpenBenchmark is a platform for turning agent trajectories into custom benchmarks and ranking models on score, cost, and speed. It allows importing traces, drafting tasks with an AI agent, and running evaluations across models.","whyItMatters":"This platform enables organizations to create benchmarks from their own data, potentially improving model selection for specific tasks. However, it is a tool for building benchmarks rather than a defined benchmark itself.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T09:02:46.503495Z","inputHash":"c0dffd8f2af7e372460ba2fea9c10bb8c3b3f4e233eef3e3d8eaff7206aca41a"},"motivation":"OpenBenchmark Turn real agent trajectories into runnable benchmarks, then rank models, harnesses, and your own agents on score, cost, and speed.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T06:56:13.762703Z","model":"deepseek-v4-pro","decisionReason":"The repository provides a fully functional system with clear instructions, scoring methods, and leaderboard generation. Users can create and run benchmarks, making it an ongoing public submission platform."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/crispyberry/OpenBenchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"score_submission","publishers":[{"name":"Pi","organizationType":"company-research-lab","sourceUrl":"https://pi.dev","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_one-success-isn-t-reliability-thinkingbox-_a377202a","familyId":"bmf_c4ef2d2a7c2c","name":"One Success Isn't Reliability: Thinkingbox, a Sandbox and Benchmark for Agents in Stateful Business Workflows","oneLine":"Evaluates agents on stateful business workflows within an MCP-compatible sandbox, using task-specific executable checks over terminal backend state to determine success.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-21","recognitionConfidence":0.7,"links":{"report":"https://arxiv.org/abs/2608.19741","pdf":"https://arxiv.org/pdf/2608.19741","project":null,"code":"https://github.com/microsoft/thinkingbox","data":null,"hfPaper":null},"evidence":{"snippet":"We release both Thinkingbox and Thinkingbox-Bench: https://github.com/microsoft/thinkingbox","reasonCodes":["benchmark term in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":17,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.19741"},"ranking":{"30d":{"score":68,"rank":33,"coverage":0.85,"confidence":"High"},"90d":{"score":63,"rank":115,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates agents on stateful business workflows within an MCP-compatible sandbox, using task-specific executable checks over terminal backend state to determine success.","whyItMatters":"Addresses the gap between occasional success and reliable completion in multi-turn tool-use tasks, where action validity alone may not reflect end-to-end correctness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:12.448463Z","inputHash":"f08a9e473a7b581a72a4d78b095c43785304e3066454634cc8df4abd22b71098"},"motivation":"Recent agent benchmarks increasingly ground evaluation in executable environments, from code repair to web navigation, app APIs, and function calling.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-21T04:30:12.448463Z","model":"deepseek-v4-flash","decisionReason":"The code and data are publicly available, enabling independent reuse. The evaluation protocol is defined by executable checks, providing a stable scoring contract."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.19741","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-21T04:30:09.569171Z"},"evaluationMode":"public_reusable","publishers":[{"name":"Microsoft","organizationType":"company-research-lab","sourceUrl":"https://github.com/microsoft/thinkingbox","role":"benchmark-publisher"}],"displayEligible":true,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_musp-bench_8c379446","familyId":"bmf_45c04f063445","name":"MuSP-Bench","oneLine":"MuSP-Bench is a 490-question benchmark for musical score understanding, performance listening, and combined score-performance reasoning. Questions cover short, mid, and long musical spans, with modalities for score (S), performance (P), both (SP), and either source (S/P).","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://huggingface.co/datasets/bryel-labs/MuSP-Bench","pdf":null,"project":null,"code":null,"data":"https://huggingface.co/datasets/bryel-labs/MuSP-Bench","hfPaper":null},"evidence":{"snippet":"MuSP-Bench MuSP-Bench is a 490-question benchmark for musical score understanding, performance listening, and combined score-performance reasoning.","reasonCodes":["discovered via huggingface","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":54,"hfDatasetLikes":0},"source":{"type":"huggingface","id":"huggingface:bryel-labs/musp-bench"},"ranking":{"30d":{"score":47,"rank":82,"coverage":0.15,"confidence":"Low","datasetDownloadRank":14,"datasetRankPopulation":20},"90d":{"score":41,"rank":263,"coverage":0.3,"confidence":"Low","datasetDownloadRank":40,"datasetRankPopulation":51}},"description":"MuSP-Bench is a 490-question benchmark for musical score understanding, performance listening, and combined score-performance reasoning. Questions cover short, mid, and long musical spans, with modalities for score (S), performance (P), both (SP), and either source (S/P).","whyItMatters":"This benchmark fills a gap in evaluating multimodal understanding of music, combining symbolic score and audio performance. It provides a standardized set with multiple input representations (ABC, MIDI, images, audio) and ablation results, useful for comparing models on music reasoning tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T09:02:46.503495Z","inputHash":"2af36117d73d2038dbaa71c05f168bb69a92815db51138c61c2c11764b4e73cb"},"motivation":"MuSP-Bench MuSP-Bench is a 490-question benchmark for musical score understanding, performance listening, and combined score-performance reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T06:56:13.762703Z","model":"deepseek-v4-pro","decisionReason":"The Hugging Face dataset page includes all questions, answers, and input files, making it a fixed public dataset with a clear evaluation protocol."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://huggingface.co/datasets/bryel-labs/MuSP-Bench","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"public_reusable","publishers":[{"name":"Bryel Labs","organizationType":"company-research-lab","sourceUrl":"https://huggingface.co/datasets/bryel-labs/MuSP-Bench","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_maliciousskillbench_08ed3dc5","familyId":"bmf_0285cbc8ba0d","name":"MaliciousSkillBench","oneLine":"MaliciousSkillBench evaluates malicious Agent Skill detection, consolidating 13 sources into 9,740 skills (7,505 malicious, 2,235 benign) across 11 attack categories. It includes protocols for Random and Source-Disjoint splits, measuring Macro-F1 and false positive rates.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-20","firstSeenAt":"2026-08-21","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.19901","pdf":"https://arxiv.org/pdf/2608.19901","project":"https://protectskills.github.io/MaliciousSkillBench/","code":null,"data":null,"hfPaper":null},"evidence":{"snippet":"We present MaliciousSkillBench, a comprehensive benchmark for malicious Agent Skill detection.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.19901"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MaliciousSkillBench evaluates malicious Agent Skill detection, consolidating 13 sources into 9,740 skills (7,505 malicious, 2,235 benign) across 11 attack categories. It includes protocols for Random and Source-Disjoint splits, measuring Macro-F1 and false positive rates.","whyItMatters":"Addresses fragmentation in malicious-skill datasets by providing a comprehensive benchmark that jointly measures attack detection and benign over-flagging. It highlights source-disjoint performance gaps, guiding development of robust detectors for real-world deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T05:19:04.530412Z","inputHash":"ecd43125f98f5f07732f9a9a30fa6c1fad55ccd3985e076fc535fb6000f09ca3"},"motivation":"Agent Skills extend LLM agents with reusable instruction packages that may also include scripts, resources, and service configuration.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-21T04:30:12.448463Z","model":"deepseek-v4-flash","decisionReason":"The project page provides official access to the benchmark, and the protocol includes defined evaluation splits and metrics."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.19901","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-21T04:30:09.569171Z"},"evaluationMode":"public_reusable","publishers":[{"name":"MaliciousSkillBench Project","organizationType":"benchmark-organization","sourceUrl":"https://protectskills.github.io/MaliciousSkillBench/","role":"benchmark-publisher"}],"displayEligible":true,"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_exphy-a-benchmark-for-explicit-physical-pr_ac5ba09f","familyId":"bmf_1779aacd1819","name":"ExPhy: A Benchmark for Explicit Physical Property Learning in Multi-Object Trajectory Forecasting","oneLine":"ExPhy is a multi-object trajectory forecasting benchmark with 24,000 simulated scenes and explicit labels for mass, friction, and restitution. It includes ID and OOD splits and supports joint evaluation of forecasting and physical property estimation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-21","recognitionConfidence":0.7,"links":{"report":"https://arxiv.org/abs/2608.20009","pdf":"https://arxiv.org/pdf/2608.20009","project":null,"code":"https://github.com/Zest86/ExPhy","data":null,"hfPaper":null},"evidence":{"snippet":"To address this gap, we introduce \\emph{ExPhy}, a multi-object trajectory forecasting benchmark containing 24,000 simulated physical scenes with explicit object-level labels for mass, friction, and restitution.","reasonCodes":["benchmark term in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.20009"},"ranking":{"30d":{"score":34,"rank":119,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":342,"coverage":0.55,"confidence":"Low"}},"description":"ExPhy is a multi-object trajectory forecasting benchmark with 24,000 simulated scenes and explicit labels for mass, friction, and restitution. It includes ID and OOD splits and supports joint evaluation of forecasting and physical property estimation.","whyItMatters":"It allows evaluation of physical property recovery alongside trajectory prediction, revealing that accurate forecasting does not guarantee accurate property estimation, which is important for physics-based reasoning in models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:12.448463Z","inputHash":"4cc10fd55119edbb381c8b941af5dd18d7630e683c980ae5456dc67c8c47bc4e"},"motivation":"Understanding object dynamics requires not only predicting future trajectories but also examining whether a model captures the physical properties that govern motion.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-21T04:30:12.448463Z","model":"deepseek-v4-flash","decisionReason":"The benchmark is released with code and data on GitHub, providing a clear public path for reuse and inspection."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.20009","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-21T04:30:09.569171Z"},"evaluationMode":"public_reusable","publishers":[{"name":"Beijing University of Posts and Telecommunications","organizationType":"academic-lab","sourceUrl":"https://github.com/Zest86/ExPhy","role":"benchmark-publisher"}],"displayEligible":true,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_euskal-agent-bench_06279148","familyId":"bmf_f335d1b163a1","name":"euskal-agent-bench","oneLine":"euskal-agent-bench evaluates LLM agents in Basque, Spanish, and English on a 30-task battery of extraction, reasoning, tool use, and format compliance. Tasks are presented in each language with localized output contracts and tools, and performance is measured on accuracy, contract compliance, and tool-call validity.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/GorkaFS13/euskal-agent-bench","pdf":null,"project":"https://gorkafs13.github.io/euskal-agent-bench/","code":"https://github.com/GorkaFS13/euskal-agent-bench","data":null,"hfPaper":null},"evidence":{"snippet":"euskal-agent-bench Do LLM agents work in Basque?","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:gorkafs13/euskal-agent-bench"},"ranking":{"30d":{"score":50,"rank":69,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":278,"coverage":0.55,"confidence":"Low"}},"description":"euskal-agent-bench evaluates LLM agents in Basque, Spanish, and English on a 30-task battery of extraction, reasoning, tool use, and format compliance. Tasks are presented in each language with localized output contracts and tools, and performance is measured on accuracy, contract compliance, and tool-call validity.","whyItMatters":"This benchmark addresses the gap in evaluating agentic capabilities in low-resource languages, specifically Basque. It measures not only task accuracy but also the robustness of agentic infrastructure (output contracts, tool use) under language shift, which is critical for deploying LLM systems in multilingual contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-23T09:02:46.503495Z","inputHash":"b0a4597b12e4689ed6baa0a624cfba347be14226009872e5ae522f1453ab82fd"},"motivation":"euskal-agent-bench Do LLM agents work in Basque?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T06:56:13.762703Z","model":"deepseek-v4-pro","decisionReason":"The benchmark includes a public code repository, live leaderboard, and defined scoring metrics across task types, supporting ongoing model comparisons."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/GorkaFS13/euskal-agent-bench","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"score_submission","publishers":[{"name":"GorkaFS13","organizationType":"community","sourceUrl":"https://github.com/GorkaFS13/euskal-agent-bench","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_dgx-spark-llm-lab_ca498b49","familyId":"bmf_cee898b42505","name":"dgx-spark-llm-lab","oneLine":"A benchmark for comparing LLM setups—model, quantization, serving flags, and thinking mode—on the user's own hardware. It runs hidden unit-test-based evaluation suites covering code generation and agentic tasks, and provides a command to apply the winning configuration.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/luongnv89/dgx-spark-llm-lab","pdf":null,"project":"https://img.shields.io/badge/license-MIT-blue","code":"https://github.com/luongnv89/dgx-spark-llm-lab","data":null,"hfPaper":null},"evidence":{"snippet":"Benchmark model + quant + serving flags + thinking mode on your own hardware, then install the config that won.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:luongnv89/dgx-spark-llm-lab"},"ranking":{"30d":{"score":34,"rank":120,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":343,"coverage":0.55,"confidence":"Low"}},"description":"A benchmark for comparing LLM setups—model, quantization, serving flags, and thinking mode—on the user's own hardware. It runs hidden unit-test-based evaluation suites covering code generation and agentic tasks, and provides a command to apply the winning configuration.","whyItMatters":"Public leaderboards often rank models on hardware and settings that differ from real deployments. This benchmark lets users evaluate setups on their own endpoint, providing actionable configuration choices. It also reveals trade-offs between thinking mode, cost, and task type.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:07:02.538621Z","inputHash":"827733a50630a807d431ded2ec5e6d9e3e44f8ea9dab35c13b355c51d9a5a942"},"motivation":"dgx-spark-llm-lab Not a leaderboard — a way to pick your daily driver.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T09:01:42.984229Z","model":"deepseek-v4-pro","decisionReason":"The repository provides a clear evaluation protocol with validated hidden tests, repeated campaign reports, and installable configuration artifacts, enabling other teams to run and compare local setups."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/luongnv89/dgx-spark-llm-lab","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"public_reusable","publishers":[{"name":"luongnv89","organizationType":"community","sourceUrl":"https://github.com/luongnv89/dgx-spark-llm-lab","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_deltaml-bench_b6518e0f","familyId":"bmf_dcd3591b7e50","name":"DeltaML-Bench","oneLine":"DeltaML-Bench evaluates ML agents on 48 tasks from research papers, requiring improvement of published baselines within real repositories. It measures per-run success rate and specification gaming across configurations with time allocations (e.g., 4x6h, 2x12h).","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-21","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.19653","pdf":"https://arxiv.org/pdf/2608.19653","project":null,"code":"https://github.com/AlgorithmicResearchGroup/deltaml-bench-public","data":null,"hfPaper":null},"evidence":{"snippet":"We introduce DeltaML-Bench, a benchmark comprising 48 tasks sourced from research papers that require agents to improve published baselines within imperfect, open-source repositories.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.19653"},"ranking":{"30d":{"score":34,"rank":121,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":344,"coverage":0.55,"confidence":"Low"}},"description":"DeltaML-Bench evaluates ML agents on 48 tasks from research papers, requiring improvement of published baselines within real repositories. It measures per-run success rate and specification gaming across configurations with time allocations (e.g., 4x6h, 2x12h).","whyItMatters":"Benchmarks agents under realistic compute constraints and detects specification gaming, which is critical for trustworthy autonomous ML experimentation. Results show scaffolding design (ARG) improves success and reduces gaming, informing agent development practices.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T05:19:04.530412Z","inputHash":"ee41b629feb8f976a34d4eea0c2d8cb6c5dd380d382a470be4968c092d3f3585"},"motivation":"Autonomous agents for machine learning experimentation must navigate heterogeneous repositories, repair training pipelines, and evaluate candidate improvements under realistic compute constraints.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-21T04:30:12.448463Z","model":"deepseek-v4-flash","decisionReason":"The benchmark is publicly available with code and grading utilities on GitHub, and it provides a clear protocol for evaluating agents."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.19653","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-21T04:30:09.569171Z"},"evaluationMode":"score_submission","publishers":[{"name":"Algorithmic Research Group","organizationType":"academic-lab","sourceUrl":"https://github.com/AlgorithmicResearchGroup/deltaml-bench-public","role":"benchmark-publisher"}],"displayEligible":true,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_amo-bench-p_ed92afee","familyId":"bmf_bc43e071f49e","name":"amo-bench-p","oneLine":"A 39-problem subset of AMO-Bench for mathematical reasoning, filtered to questions with parser-gradable answer types (number, set, variable). It relies on the official AMO-Bench evaluator for answer extraction and verification, providing a fixed dataset for evaluating reasoning models.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://huggingface.co/datasets/djalexj/amo-bench-p","pdf":null,"project":null,"code":null,"data":"https://huggingface.co/datasets/djalexj/amo-bench-p","hfPaper":null},"evidence":{"snippet":"AMO-Bench-P This is the 39-problem, parser-graded subset of meituan-longcat/AMO-Bench.","reasonCodes":["discovered via huggingface","benchmark term in abstract","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":78,"hfDatasetLikes":0},"source":{"type":"huggingface","id":"huggingface:djalexj/amo-bench-p"},"ranking":{"30d":{"score":51,"rank":62,"coverage":0.15,"confidence":"Low","datasetDownloadRank":10,"datasetRankPopulation":20},"90d":{"score":46,"rank":230,"coverage":0.3,"confidence":"Low","datasetDownloadRank":34,"datasetRankPopulation":51}},"description":"A 39-problem subset of AMO-Bench for mathematical reasoning, filtered to questions with parser-gradable answer types (number, set, variable). It relies on the official AMO-Bench evaluator for answer extraction and verification, providing a fixed dataset for evaluating reasoning models.","whyItMatters":"This subset enables reproducible evaluation of mathematical reasoning without the cost and variability of LLM-based grading. It offers a clear, automated scoring protocol, making it useful for rapid model comparison and regression testing.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:07:02.538621Z","inputHash":"8daf445b64e0fe0920655299dee5ace307877432198e821c996ffd4b338db332"},"motivation":"AMO-Bench-P This is the 39-problem, parser-graded subset of meituan-longcat/AMO-Bench.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T09:01:42.984229Z","model":"deepseek-v4-pro","decisionReason":"The dataset is released on Hugging Face with a clear filtering rationale and preserves the upstream evaluator, making it reusable for parser-based scoring with a stable contract."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://huggingface.co/datasets/djalexj/amo-bench-p","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"public_reusable","publishers":[{"name":"djalexj","organizationType":"community","sourceUrl":"https://huggingface.co/datasets/djalexj/amo-bench-p","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_alzheimer-classification-benchmark_abb5f538","familyId":"bmf_41ba03d9f703","name":"alzheimer-classification-benchmark","oneLine":"A benchmark comparing 11 machine learning classifiers for Alzheimer's disease detection on a synthetic dataset. It defines a fixed preprocessing, feature selection, cross-validation, and evaluation pipeline in KNIME and Python, with sensitivity as the primary metric.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences","Robotics & Autonomous Systems"],"primaryDomain":"Health & Life Sciences","industrySectors":["Software & Cloud","Pharma & Biotech","Robotics"],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/hervemottaran/alzheimer-classification-benchmark","pdf":null,"project":"https://img.shields.io/badge/KNIME%20Analytics%20Platform-5.9-FDD800?logo=knime&logoColor=000000","code":"https://github.com/hervemottaran/alzheimer-classification-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"alzheimer-classification-benchmark Benchmarking 11 classifiers for Alzheimer's disease detection on synthetic health data with KNIME and Python.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:hervemottaran/alzheimer-classification-benchmark"},"ranking":{"30d":{"score":34,"rank":122,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":345,"coverage":0.55,"confidence":"Low"}},"description":"A benchmark comparing 11 machine learning classifiers for Alzheimer's disease detection on a synthetic dataset. It defines a fixed preprocessing, feature selection, cross-validation, and evaluation pipeline in KNIME and Python, with sensitivity as the primary metric.","whyItMatters":"This benchmark provides a reproducible comparison of classifiers for a medical screening task, using a public dataset. It offers insights into algorithm trade-offs on a fixed protocol, useful for practitioners selecting models for similar classification problems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:07:02.538621Z","inputHash":"22a59e6bd8dd18d79301d6aae65d61b078bf97bf6c74f47879b07de08c6a440a"},"motivation":"alzheimer-classification-benchmark Benchmarking 11 classifiers for Alzheimer's disease detection on synthetic health data with KNIME and Python.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T09:01:42.984229Z","model":"deepseek-v4-pro","decisionReason":"The repository defines a repeatable pipeline with data, workflow, and evaluation metrics, and is publicly available for others to run or inspect."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/hervemottaran/alzheimer-classification-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"public_reusable","publishers":[{"name":"hervemottaran","organizationType":"community","sourceUrl":"https://github.com/hervemottaran/alzheimer-classification-benchmark","role":"benchmark-publisher"}],"displayEligible":false,"capabilityGroups":["Multimodal Perception","Coding & Software Engineering"],"domainScope":"cross-domain"},{"id":"bm_ai4ai-bench_cbcf9954","familyId":"bmf_300170ad25dc","name":"AI4AI-Bench","oneLine":"A benchmark for evaluating LLM agents on algorithmic design tasks for recursive self-improvement. It includes 10 frozen research repositories across 10 training algorithm families, with agents given 4 hours to rewrite training algorithms and scored by hidden evaluators on a normalized scale.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Self-Evolution"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.20318","pdf":"https://arxiv.org/pdf/2608.20318","project":null,"code":null,"data":null,"hfPaper":null},"evidence":{"snippet":"We present AI4AI\\mbox{-}Bench, 10 frozen research repositories spanning 10 training algorithm families.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.20318"},"ranking":{"30d":{"score":47,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"A benchmark for evaluating LLM agents on algorithmic design tasks for recursive self-improvement. It includes 10 frozen research repositories across 10 training algorithm families, with agents given 4 hours to rewrite training algorithms and scored by hidden evaluators on a normalized scale.","whyItMatters":"Existing benchmarks do not isolate the ability to design training algorithms, which is crucial for recursive self-improvement. This benchmark provides a standardized evaluation for such agents, enabling comparison and tracking progress as systems evolve.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:07:02.538621Z","inputHash":"c514b352b1ccb9b8fe71b0f8b5bc317f282547e343129fe336603fb563ee4fe2"},"motivation":"Recursive self-improvement (RSI) asks whether an AI system can improve the process that produces AI systems, so that the next system inherits the improvement.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T09:01:42.984229Z","model":"deepseek-v4-pro","decisionReason":"The benchmark is named, defines a repeatable evaluation object (10 frozen repositories, fixed procedure, hidden evaluator) and a comparable scoring contract (normalized scale from uninformative model to task optimum). The paper explicitly states the task suite, evaluators, and every scored submission are released, providing a public path for other teams to run or inspect it."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.20318","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:50.392548Z"},"evaluationMode":"score_submission","displayEligible":true,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_ai-gateway-reproducible-benchmark_99ff14f3","familyId":"bmf_7fb20c44ab0e","name":"ai-gateway-reproducible-benchmark","oneLine":"A benchmark that compares AI gateway solutions (GoModel, LiteLLM, Portkey, Bifrost) for LLM serving. It measures latency, throughput, memory, cold start, and image size on a fixed AWS instance, using a one-command reproducible setup.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-20","firstSeenAt":"2026-08-23","recognitionConfidence":0.4,"links":{"report":"https://github.com/ENTERPILOT/ai-gateway-reproducible-benchmark","pdf":null,"project":null,"code":"https://github.com/ENTERPILOT/ai-gateway-reproducible-benchmark","data":null,"hfPaper":null},"evidence":{"snippet":"ai-gateway-reproducible-benchmark Comparing Bifrost vs LiteLLM vs GoModel vs PortkeyAI.","reasonCodes":["discovered via github","benchmark term in abstract","evaluation protocol evidence","public artifact URL","no explicit benchmark release evidence"]},"dataStatus":"primary-source-candidate","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"github","id":"github:enterpilot/ai-gateway-reproducible-benchmark"},"ranking":{"30d":{"score":34,"rank":123,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":346,"coverage":0.55,"confidence":"Low"}},"description":"A benchmark that compares AI gateway solutions (GoModel, LiteLLM, Portkey, Bifrost) for LLM serving. It measures latency, throughput, memory, cold start, and image size on a fixed AWS instance, using a one-command reproducible setup.","whyItMatters":"This benchmark provides transparent, reproducible performance comparisons for AI gateways, helping teams choose the right infrastructure. It tracks metrics over time, offering practical decision support for deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-24T07:07:02.538621Z","inputHash":"a5af503d7eac6baaa8559727190dd3f16e60c50f26eb1ab15689d6fa9ac1175b"},"motivation":"ai-gateway-reproducible-benchmark Comparing Bifrost vs LiteLLM vs GoModel vs PortkeyAI.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-23T09:01:42.984229Z","model":"deepseek-v4-pro","decisionReason":"The repository provides a named benchmark with a stable scoring contract, clear public reuse path via a runnable AWS harness, and ongoing result tracking, making it suitable for model comparison."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://github.com/ENTERPILOT/ai-gateway-reproducible-benchmark","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-23T06:55:51.988821Z"},"evaluationMode":"score_submission","displayEligible":false,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_va-judger-bench_7e8bd378","familyId":"bmf_52e48663243f","name":"VA-Judger-Bench","oneLine":"VA-Judger-Bench evaluates reward models for joint video-audio generation. It contains paired comparisons of generated video-audio samples with human preference labels, covering in-domain and out-of-domain model outputs.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-19","firstSeenAt":"2026-08-20","recognitionConfidence":0.55,"links":{"report":"https://arxiv.org/abs/2608.18607","pdf":"https://arxiv.org/pdf/2608.18607","project":null,"code":"https://github.com/ShareLab-SII/VA-Judger","data":null,"hfPaper":null},"evidence":{"snippet":"We also introduce the VA-Judger-Bench benchmark with both in-domain and out-of-domain model comparisons to evaluate whether reward models truly align with human preferences.","reasonCodes":["exact named benchmark artifact released in abstract","evaluation protocol evidence","public artifact URL","named entity is a model, method, framework, or dataset used in evaluation"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":11,"hfDailySubmittedAt":"2026-08-20T00:00:00.000Z","githubStars":51,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.18607"},"ranking":{"30d":{"score":80,"rank":11,"coverage":0.85,"confidence":"High"},"90d":{"score":75,"rank":48,"coverage":0.7,"confidence":"Medium"}},"description":"VA-Judger-Bench evaluates reward models for joint video-audio generation. It contains paired comparisons of generated video-audio samples with human preference labels, covering in-domain and out-of-domain model outputs.","whyItMatters":"Existing metrics evaluate quality dimensions separately, missing semantic and temporal coherence. VA-Judger-Bench provides a benchmark to assess whether reward models align with human preferences for joint video-audio generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f04b475f5407961dea11eea659358dca79197764b37006841a6c009cabb9b383"},"motivation":"Using reinforcement learning to post-train joint video-audio generation models requires a reward signal.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-20T09:45:58.504563Z","model":"deepseek-v4-flash","decisionReason":"VA-Judger-Bench is a named benchmark with a defined evaluation protocol, publicly released data/checkpoints, and clear scoring contract for comparing reward models."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.18607","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-20T09:44:47.308583Z"},"evaluationMode":"public_reusable","publishers":[{"name":"ShareLab-SII","organizationType":"academic-lab","sourceUrl":"https://github.com/ShareLab-SII/VA-Judger","role":"benchmark-publisher"}],"displayEligible":true,"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_softvtbench-a-deformation-aware-visuo-tact_61935c97","familyId":"bmf_3799d0e658b4","name":"SoftVTBench: A Deformation-Aware Visuo-Tactile Dataset and Benchmark for Deformable-Object Manipulation","oneLine":"SoftVTBench is a visuo-tactile dataset and benchmark for deformable-object manipulation, providing expert demonstrations with synchronized sensor data and finite-element ground truth, with a closed-loop evaluation protocol.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-19","firstSeenAt":"2026-08-20","recognitionConfidence":0.55,"links":{"report":"https://arxiv.org/abs/2608.18701","pdf":"https://arxiv.org/pdf/2608.18701","project":null,"code":null,"data":null,"hfPaper":null},"evidence":{"snippet":"We introduce SoftVTBench, a visuo-tactile dataset for physical-interaction-aware deformable-object manipulation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","evaluation protocol evidence","named entity is a model, method, framework, or dataset used in evaluation"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":12,"hfDailySubmittedAt":"2026-08-20T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.18701"},"ranking":{"30d":{"score":56,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":54,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SoftVTBench is a visuo-tactile dataset and benchmark for deformable-object manipulation, providing expert demonstrations with synchronized sensor data and finite-element ground truth, with a closed-loop evaluation protocol.","whyItMatters":"Most manipulation benchmarks evaluate task success alone, ignoring physical interaction quality. SoftVTBench enables evaluation of deformation-aware success, addressing how policies interact with deformable objects.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ea71c1b8664f5b0041da449ff541a210c76f0af48b8340723f17e962e6e421bd"},"motivation":"Physical interaction quality is central to deformable-object manipulation, yet most benchmarks evaluate task success alone.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-20T09:45:58.504563Z","model":"deepseek-v4-flash","decisionReason":"Named dataset and benchmark with clear evaluation protocol and public data availability, though code is not provided but dataset is accessible."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.18701","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-20T09:44:47.308583Z"},"evaluationMode":"public_reusable","displayEligible":true,"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_odineval_a7103a7e","familyId":"bmf_3047befaf9e9","name":"OdinEval","oneLine":"OdinEval is a benchmark for program repair in the Odin programming language, built from documented defects, with issue-to-commit bindings, regression tests, and execution records.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation"],"topics":["cs.SE"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-19","firstSeenAt":"2026-08-20","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.18595","pdf":"https://arxiv.org/pdf/2608.18595","project":null,"code":null,"data":null,"hfPaper":null},"evidence":{"snippet":"We present OdinEval, a reproducible benchmark built from documented defects in public Odin repositories.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.18595"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OdinEval is a benchmark for program repair in the Odin programming language, built from documented defects, with issue-to-commit bindings, regression tests, and execution records.","whyItMatters":"Existing repair benchmarks focus on mainstream languages, leaving systems languages like Odin untested. OdinEval provides a reproducible benchmark for a less-covered language.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6618ab0cd023268560fb59d1e91e20569a6dddb7e88ba6a6565cae82d884a0dd"},"motivation":"Repository-level repair benchmarks still center on a few mainstream languages, leaving systems languages such as Odin largely untested.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-20T09:45:58.504563Z","model":"deepseek-v4-flash","decisionReason":"Named benchmark with clear release artifacts and reproducible protocol, including frozen data and containers."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.18595","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-20T09:44:47.308583Z"},"evaluationMode":"public_reusable","displayEligible":true,"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_greekbarretrieval_75c379ac","familyId":"bmf_af9bde98b8ea","name":"GreekBarRetrieval","oneLine":"GreekBarRetrieval is a retrieval benchmark for Greek statutory articles, comprising bar-exam questions with case facts and candidate articles.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.IR"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-19","firstSeenAt":"2026-08-20","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.18752","pdf":"https://arxiv.org/pdf/2608.18752","project":null,"code":null,"data":null,"hfPaper":null},"evidence":{"snippet":"We introduce GreekBarRetrieval, a public retrieval benchmark derived from, and complementing GreekBarBench, which did not include retrieval.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.18752"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GreekBarRetrieval is a retrieval benchmark for Greek statutory articles, comprising bar-exam questions with case facts and candidate articles.","whyItMatters":"Statutory retrieval for Greek is underexplored, and this benchmark provides a testbed for retrieval methods in a low-resource legal domain.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"685bc5dad810894c79f23dd3e4b5ff645c3966a7a3ada1db977041a2b8e650b5"},"motivation":"Statutory retrieval is necessary for citation-grounded legal question answering, but remains underexplored for Greek.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-20T09:45:58.504563Z","model":"deepseek-v4-flash","decisionReason":"Provides a public benchmark with defined data, protocol, and evaluation metrics; includes baseline experiments and is accessible via ArXiv. The paper does not specify an ongoing scoring service, but the dataset and protocol are reusable."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.18752","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-20T09:44:47.308583Z"},"evaluationMode":"public_reusable","displayEligible":true,"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_fm-bench_85cc4c12","familyId":"bmf_9ecdad9645f6","name":"FM-Bench","oneLine":"FM-Bench evaluates LLM agents on long-horizon football club management through 20 in-game years with 26 tools, measuring managerial decision quality via a deterministic engine scoring.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-19","firstSeenAt":"2026-08-20","recognitionConfidence":0.75,"links":{"report":"https://arxiv.org/abs/2608.18423","pdf":"https://arxiv.org/pdf/2608.18423","project":null,"code":"https://github.com/Analogy-AI/fm-bench","data":null,"hfPaper":null},"evidence":{"snippet":"FM-Bench (Football Management Benchmark) measures this.","reasonCodes":["coined title prefix ending in Bench or Benchmark","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":16,"hfDailySubmittedAt":"2026-08-20T00:00:00.000Z","githubStars":16,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.18423"},"ranking":{"30d":{"score":77,"rank":15,"coverage":0.85,"confidence":"High"},"90d":{"score":66,"rank":93,"coverage":0.7,"confidence":"Medium"}},"description":"FM-Bench evaluates LLM agents on long-horizon football club management through 20 in-game years with 26 tools, measuring managerial decision quality via a deterministic engine scoring.","whyItMatters":"Current agent benchmarks focus on bounded tasks, leaving long-horizon decision-making unmeasured. FM-Bench provides a reproducible environment with deterministic scoring to test sustained decision quality over hundreds of steps.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6e16420e979f7ab35f80e1fb00394624134919f5a8457b2c8552cb1dfa8686c2"},"motivation":"Language model agents now execute bounded tasks reliably.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-20T09:45:58.504563Z","model":"deepseek-v4-flash","decisionReason":"Explicitly designed as a benchmark with clear scoring, code availability, and instructions for running evaluations. The abstract and code repo confirm a public reuse path and ongoing scoring via Arena."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.18423","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-20T09:44:47.308583Z"},"evaluationMode":"score_submission","publishers":[{"name":"Analogy AI","organizationType":"company-research-lab","sourceUrl":"https://github.com/Analogy-AI/fm-bench","role":"benchmark-publisher"}],"displayEligible":true,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_finrca-bench_2d78ed7b","familyId":"bmf_61689bc0d835","name":"FinRCA-Bench","oneLine":"FinRCA-Bench is a synthetic benchmark for financial reconciliation root-cause analysis, with 2,250 cases across 14 tables and 15 failure categories, providing evidence retrieval and reasoning evaluation.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":["Reasoning","Information retrieval"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-08-19","firstSeenAt":"2026-08-20","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.18534","pdf":"https://arxiv.org/pdf/2608.18534","project":null,"code":"https://github.com/PratikGhawate/FinRCA-AI-Bench","data":null,"hfPaper":null},"evidence":{"snippet":"We introduce FinRCA-Bench, a deterministic synthetic benchmark of 2,250 accounts-payable-to-bank reconciliation cases spanning 14 operational tables, including 1,500 injected failures across 15 causal categories and 750 legitimate or hard-negative cases.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.18534"},"ranking":{"30d":{"score":34,"rank":124,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":347,"coverage":0.55,"confidence":"Low"}},"description":"FinRCA-Bench is a synthetic benchmark for financial reconciliation root-cause analysis, with 2,250 cases across 14 tables and 15 failure categories, providing evidence retrieval and reasoning evaluation.","whyItMatters":"Financial AI systems need reliable evidence retrieval and reasoning, and existing benchmarks conflate the two. FinRCA-Bench separates retrieval evaluation from answer correctness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fc782ed4e27afbb14d3ccec757b5ed7ad9aa43f8750cec2b0e275d4d55ed415d"},"motivation":"Large language models are increasingly used to support financial operations, but their apparent reasoning performance can depend on whether they receive the right evidence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-20T09:45:58.504563Z","model":"deepseek-v4-flash","decisionReason":"Provides a public dataset and generation code, with clear evaluation labels and a documented protocol. The GitHub link and paper describe the benchmark's structure and use, making it reusable."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.18534","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-20T09:44:47.308583Z"},"evaluationMode":"public_reusable","publishers":[{"name":"Pratik Ghawate","organizationType":"academic-lab","sourceUrl":"https://github.com/PratikGhawate/FinRCA-AI-Bench","role":"benchmark-publisher"}],"displayEligible":true,"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"specific"},{"id":"bm_atmoscoder-bench_24ddfa7c","familyId":"bmf_5582793e0564","name":"AtmosCoder-Bench","oneLine":"AtmosCoder-Bench is an execution-grounded benchmark for LLMs on atmospheric science computation, with 436 problems and 3,910 variants, grading by executing code solutions against ground truth.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-19","firstSeenAt":"2026-08-20","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.18726","pdf":"https://arxiv.org/pdf/2608.18726","project":null,"code":"https://github.com/acodercat/AtmosCoder-Bench","data":null,"hfPaper":null},"evidence":{"snippet":"Here we introduce AtmosCoder-Bench, an execution-grounded benchmark that makes the calculation process visible.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.18726"},"ranking":{"30d":{"score":56,"rank":50,"coverage":0.55,"confidence":"Low"},"90d":{"score":44,"rank":240,"coverage":0.55,"confidence":"Low"}},"description":"AtmosCoder-Bench is an execution-grounded benchmark for LLMs on atmospheric science computation, with 436 problems and 3,910 variants, grading by executing code solutions against ground truth.","whyItMatters":"Existing evaluations score final answers, overlooking calculation process. AtmosCoder-Bench makes process visible, revealing failures in applying formulas and adapting to regimes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"024f39fc515d74250d85c364f676811509d1beaf126fe6dc0035b30611fd08e9"},"motivation":"Large language models are increasingly used for quantitative work in the environmental sciences, yet existing evaluations score only final answers, leaving calculation process unobserved.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"ai-reviewed","reviewedAt":"2026-08-20T09:45:58.504563Z","model":"deepseek-v4-flash","decisionReason":"The benchmark defines a repeatable evaluation object with a clear scoring contract (execution-grounded, unit-aware grading), publicly available code and result data, and a credible path for other teams to run it."},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.18726","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-20T09:44:47.308583Z"},"evaluationMode":"score_submission","publishers":[{"name":"acodercat","organizationType":"community","sourceUrl":"https://github.com/acodercat/AtmosCoder-Bench","role":"benchmark-publisher"}],"displayEligible":true,"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_startupbench_c9f7419c","familyId":"bmf_064eca785918","name":"StartupBench","oneLine":"Evaluates whether general-purpose agents can complete market-validated, end-to-end professional workflows and deliver usable work products.","area":"Agents & Tool Use","applicationDomains":["Finance & Economics","Science & Research"],"primaryDomain":"Finance & Economics","industrySectors":[],"capabilities":["Long-horizon task execution","Tool use","Complex instruction following","Professional artifact generation"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.17800","pdf":"https://arxiv.org/pdf/2608.17800","project":"https://startupbench.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.17800"},"evidence":{"snippet":"We introduce \\textbf{StartupBench}, an E2E agent benchmark grounded in market-validated AI startup products.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":9,"hfDailySubmittedAt":"2026-08-19T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.17800"},"ranking":{"30d":{"score":54,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":53,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"StartupBench is a benchmark for general-purpose agents on end-to-end workflows derived from market-validated AI startup products, with deliverable-oriented tasks and fine-grained rubrics.","whyItMatters":"Existing agent benchmarks are researcher-selected, leaving real-world task performance uncertain. StartupBench measures agents on tasks with demonstrated demand.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"36c7ed0bbf540064241e50febdc82ed09641f45fd042dd87433ec946514c5077"},"motivation":"Recent advances in Large Language Models(LLMs) and agents have substantially improved the ability of AI systems to execute complex tasks.","constructionDetail":"StartupBench measures whether agents can complete realistic professional workflows and produce usable final work products across six broad disciplines.","detail":{"taskBreakdown":["Medical & Healthcare","Finance","Legal","Business & Management","STEM & Computer Science","Education & Humanities"],"protocol":{"tasks":"97 end-to-end workflow tasks","primaryMetric":"Importance-weighted rubric score (0–100); success rate at score ≥90","version":"arXiv v1"},"leaderboardUrl":"https://startupbench.github.io/#leaderboard"},"curation":{"state":"source-reviewed","reviewedAt":"2026-08-20","sources":["https://arxiv.org/abs/2608.17800","https://arxiv.org/html/2608.17800","https://startupbench.github.io/"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.17800","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"evaluationMode":"score_submission","availability":{"paperStatus":"available","githubStatus":"not_found","hfDatasetStatus":"not_found","evaluatorStatus":"described_not_released","submissionStatus":"not_found"},"capabilityGroups":["Agents","Tool Calling"],"domainScope":"cross-domain"},{"id":"bm_ptxbench_11575102","familyId":"bmf_dc9322ec40cc","name":"PTXBench","oneLine":"Evaluates whether LLM agents can generate correct, architecture-specific GPU kernels that execute required PTX instructions and outperform libraries.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation","GPU kernel optimization","Architecture-specific programming","Iterative code repair"],"topics":["Code"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.17379","pdf":"https://arxiv.org/pdf/2608.17379","project":"https://github.com/zhang677/PTXBench","code":"https://github.com/zhang677/PTXBench","data":"https://huggingface.co/datasets/AccRL/accrl-training","hfPaper":"https://huggingface.co/papers/2608.17379"},"evidence":{"snippet":"We introduce PTXBench, a benchmark for evaluating and adapting large language models (LLMs) to use architecture-specific PTX for GPU kernel optimization.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":"2026-08-19T00:00:00.000Z","githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":11,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2608.17379"},"ranking":{"30d":{"score":27,"rank":141,"coverage":1.0,"confidence":"High","datasetDownloadRank":20,"datasetRankPopulation":20},"90d":{"score":16,"rank":397,"coverage":1.0,"confidence":"High","datasetDownloadRank":51,"datasetRankPopulation":51}},"description":"PTXBench evaluates LLMs in generating architecture-specific PTX for GPU kernel optimization. It measures functional correctness, execution of target instructions, and speedup over frontier libraries across GEMM and attention workloads on H100 and B200 GPUs.","whyItMatters":"Addresses the lack of standardized evaluation for LLM-driven GPU kernel optimization with architecture-specific PTX, providing a reproducible testbed to compare model capabilities and guide improvements in exploiting evolving GPU architectures.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"973bda730a80d543c43a5fee47f05df482a9a432407dfbc4a14ec612f974a73a"},"motivation":"We introduce PTXBench, a benchmark for evaluating and adapting large language models (LLMs) to use architecture-specific PTX for GPU kernel optimization.","constructionDetail":"PTXBench tests architecture-specific GPU kernel generation on H100 and B200 hardware, including correctness, required-instruction execution and speedup.","detail":{"taskBreakdown":["GEMM","Multi-head attention forward","Causal attention forward","Multi-head attention backward","Causal attention backward"],"protocol":{"tasks":"5 primary BF16 workloads plus generalization workloads","primaryMetric":"Correctness, target-instruction correctness, best speedup and Fast_p","language":"CUDA C++ with inline PTX","version":"arXiv v1"}},"curation":{"state":"source-reviewed","reviewedAt":"2026-08-20","sources":["https://arxiv.org/abs/2608.17379","https://arxiv.org/html/2608.17379","https://github.com/zhang677/PTXBench","https://huggingface.co/datasets/AccRL/accrl-training"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.17379","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"evaluationMode":"score_submission","availability":{"paperStatus":"available","githubStatus":"available","hfDatasetStatus":"available","evaluatorStatus":"available","submissionStatus":"not_found"},"publishers":[{"name":"PTXBench Team","organizationType":"academic-lab","sourceUrl":"https://github.com/zhang677/PTXBench","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_prism_da9ef810","familyId":"bmf_bb4c8eb92b6e","name":"PRISM","oneLine":"Provides multimodal robot demonstrations and real-world evaluation tasks for precision, contact-rich industrial manipulation.","area":"Robotics & Embodied AI","applicationDomains":["Industrial & Engineering","Robotics & Autonomous Systems"],"primaryDomain":"Industrial & Engineering","industrySectors":["Manufacturing","Robotics"],"capabilities":["Multimodal robot manipulation","Contact-rich control","Imitation learning","Force-aware manipulation"],"topics":["Robotics","Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-18","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.17962","pdf":"https://arxiv.org/pdf/2608.17962","project":"https://tengbo-yu.github.io/PRISM/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.17962"},"evidence":{"snippet":"In contrast to datasets collected in household or laboratory settings, PRISM provides a realistic benchmark for multimodal perception and control under high-precision industrial constraints, and serves as a foundation for contact-rich, generalizable manipulation in real-world manufacturing environments.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.17962"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PRISM is a dataset of over 5,000 trajectories across 25 industrial manipulation tasks with multimodal sensing. It provides teleoperated demonstrations for contact-rich operations but lacks a standardized evaluation protocol or scoring contract.","whyItMatters":"As a dataset, it could support research in contact-rich manipulation, but without a scoring mechanism it does not constitute a benchmark for model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f00b035dd00ad97000c398ba609017d1a1aebd893b7811b59f8ce360c77253c8"},"motivation":"Recent progress in robotic learning has been fueled by large-scale datasets collected in everyday environments.","constructionDetail":"PRISM covers precision industrial manipulation using synchronized RGB-D, force/torque, tactile and robot-state observations.","detail":{"taskBreakdown":["Precision contact-rich assembly","Product packaging","Dynamic object sorting","Force-aware manipulation"],"protocol":{"tasks":"25+ tasks and 5,000+ trajectories","primaryMetric":"Task success rate over 20 real-robot trials per configuration","version":"arXiv v1"}},"curation":{"state":"source-reviewed","reviewedAt":"2026-08-20","sources":["https://arxiv.org/abs/2608.17962","https://arxiv.org/html/2608.17962","https://tengbo-yu.github.io/PRISM/"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.17962","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"evaluationMode":"score_submission","availability":{"paperStatus":"available","githubStatus":"announced_not_released","hfDatasetStatus":"announced_not_released","evaluatorStatus":"described_not_released","submissionStatus":"not_found"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"cross-domain"},{"id":"bm_pathoargus-bench_2bed53e5","familyId":"bmf_9d5dfa0f7014","name":"PathoArgus-Bench","oneLine":"Evaluates evidence-grounded visual reasoning over complete gigapixel whole-slide and multi-slide pathology cases.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Medical image reasoning","Long-context reasoning","Evidence grounding"],"topics":["Long Context","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.17607","pdf":"https://arxiv.org/pdf/2608.17607","project":null,"code":null,"data":"https://huggingface.co/datasets/liubw/PathoArgus-Bench","hfPaper":"https://huggingface.co/papers/2608.17607"},"evidence":{"snippet":"We introduce PathoArgus-Bench, a benchmark and evaluation protocol that explicitly tests the full evidence chain: availability, accessibility, use, and responsiveness.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":67,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2608.17607"},"ranking":{"30d":{"score":50,"rank":78,"coverage":0.15,"confidence":"Low","datasetDownloadRank":11,"datasetRankPopulation":20},"90d":{"score":45,"rank":235,"coverage":0.3,"confidence":"Low","datasetDownloadRank":35,"datasetRankPopulation":51}},"description":"PathoArgus-Bench evaluates evidence-grounded visual reasoning in whole-slide pathology. It comprises 22,078 multiple-choice questions from 4,913 patients across 15 TCGA projects, testing availability, accessibility, use, and responsiveness of evidence under a fixed reader budget.","whyItMatters":"Addresses the gap where final answer accuracy is insufficient to establish evidence grounding in pathology AI. Provides a protocol to assess whether models truly use supplied tissue evidence, offering a more rigorous evaluation for clinical deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"459e70fb59ee84661c78fd4108f27802d708370fbf5061622b60b457c6dfdb2f"},"motivation":"Whole-slide pathology reasoning requires models to integrate gigapixel-scale visual evidence across complete case-linked slides, yet current question-answering benchmarks primarily measure final answer accuracy--a metric vulnerable to linguistic priors and benchmark regularities, and insufficient to establish that predictions are grounded in the supplied tissue.","constructionDetail":"PathoArgus-Bench measures case-level pathology reasoning over whole-slide images, including diagnostic findings, staging and exact evidence grounding.","detail":{"taskBreakdown":["Anatomic site","Histologic type","Tumor grade","Mitotic activity","Local invasion","Surgical margins","Nodal status","Pathologic staging","Cross-slide integration","Evidence grounding"],"protocol":{"tasks":"22,078 four-choice questions from 4,913 patients and 5,400 whole-slide images","primaryMetric":"Overall accuracy and QExact","version":"arXiv v1"}},"curation":{"state":"source-reviewed","reviewedAt":"2026-08-20","sources":["https://arxiv.org/abs/2608.17607","https://arxiv.org/html/2608.17607","https://huggingface.co/datasets/liubw/PathoArgus-Bench"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.17607","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"evaluationMode":"score_submission","availability":{"paperStatus":"available","githubStatus":"not_found","hfDatasetStatus":"available","evaluatorStatus":"not_found","submissionStatus":"not_found"},"publishers":[{"name":"PathoArgus-Bench Team","organizationType":"academic-lab","sourceUrl":"https://huggingface.co/datasets/liubw/PathoArgus-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception","Long Context & Memory"],"domainScope":"specific"},{"id":"bm_mobileworldsafety_b2abd1b3","familyId":"bmf_d8688a41bdaf","name":"MobileWorldSafety","oneLine":"Evaluates Android GUI-agent safety against environmental injection attacks embedded in everyday mobile workflows.","area":"Agents & Tool Use","applicationDomains":["Consumer & Productivity","Cybersecurity"],"primaryDomain":"Consumer & Productivity","industrySectors":["Consumer Technology"],"capabilities":["Computer use","Prompt-injection resistance","Mobile interaction"],"topics":["Agents","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.17659","pdf":"https://arxiv.org/pdf/2608.17659","project":"https://anonymous.4open.science/r/Anonymous_sub-C887","code":"https://anonymous.4open.science/r/Anonymous_sub-C887","data":"https://anonymous.4open.science/r/Anonymous_sub-C887","hfPaper":"https://huggingface.co/papers/2608.17659"},"evidence":{"snippet":"To address this gap, we introduce MobileWorldSafety, a benchmark of 142 risk tasks built on real Android applications.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.17659"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MobileWorldSafety evaluates GUI agents' safety against environmental injection attacks in Android apps. It includes 142 risk tasks on real applications, with programmatically verifiable risk indicators and a two-stage pipeline for verification.","whyItMatters":"Provides a quantitative measure of GUI agent vulnerability to injection attacks, enabling comparison across agents and supporting development of safer mobile agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2db901229ee0346033001fb1dcb72bf015d9f310f65a1d28077cd4fc2e1e1d70"},"motivation":"LLM-powered GUI agents that autonomously operate smartphones are rapidly transitioning from research prototypes to early real-world deployment.","constructionDetail":"MobileWorldSafety embeds environmental injection attacks into executable Android workflows and verifies both attack success and task completion.","detail":{"taskBreakdown":["Email","Messaging","Calendar","Files","Web browsing","Navigation","Collaboration","Social media","E-commerce","Tool responses"],"protocol":{"tasks":"142 risk tasks across 13 Android apps and 5 MCP servers","primaryMetric":"Attack Success Rate and Task Completion Rate under attack","version":"arXiv v1"}},"curation":{"state":"source-reviewed","reviewedAt":"2026-08-20","sources":["https://arxiv.org/abs/2608.17659","https://arxiv.org/html/2608.17659","https://anonymous.4open.science/r/Anonymous_sub-C887"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.17659","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"evaluationMode":"score_submission","availability":{"paperStatus":"available","githubStatus":"available","hfDatasetStatus":"available","evaluatorStatus":"available","submissionStatus":"not_found"},"capabilityGroups":["Agents"],"domainScope":"cross-domain"},{"id":"bm_livehouse-ts_9c17ef53","familyId":"bmf_4ac789a70586","name":"LiveHouse-TS","oneLine":"Continuously evaluates time-series foundation models on future observations from live cross-domain data streams.","area":"Language & Knowledge","applicationDomains":["Finance & Economics","Science & Research","Transport & Logistics"],"primaryDomain":"Finance & Economics","industrySectors":[],"capabilities":["Time-series forecasting","Temporal robustness","Distribution shift"],"topics":["cs.AI"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.17299","pdf":"https://arxiv.org/pdf/2608.17299","project":"https://huggingface.co/spaces/CityMindDev/LiveHouse-TS","code":"https://github.com/zhouziyu02/TS-Live","data":"https://huggingface.co/spaces/CityMindDev/LiveHouse-TS","hfPaper":"https://huggingface.co/papers/2608.17299"},"evidence":{"snippet":"To bridge this gap, we introduce LiveHouse-TS, the first open-world living benchmark infrastructure for TSFMs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.17299"},"ranking":{"30d":{"score":34,"rank":125,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":348,"coverage":0.55,"confidence":"Low"}},"description":"LiveHouse-TS is a living benchmark for time series foundation models, evaluating models prequentially on real future data across 11 domains and 17 datasets. It shifts benchmarking from snapshot accuracy to continuous temporal validity.","whyItMatters":"Addresses the limitation of static benchmarks for time series models, providing a framework to assess model robustness under distribution shifts and long-term ranking stability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"24f5f3a0203624d8edded8c57b3e2cdf654c09f127ae57d13c047ce520fbc867"},"motivation":"Time Series Foundation Models (TSFMs) have recently emerged as a highly promising paradigm for cross-domain zero-shot forecasting.","constructionDetail":"LiveHouse-TS uses newly arriving observations from 17 datasets so forecasting models are evaluated under real distribution shift rather than a frozen test set.","detail":{"taskBreakdown":["Weather","Air quality","Energy","Hydrology","Ocean","Traffic","Finance","Web attention","Macroeconomics","News events","Disaster events"],"protocol":{"tasks":"Continuous forecasting over 17 datasets from 15 sources and 11 domains","primaryMetric":"RMSE and CRPS, summarized by Average Rank, Win Rate and Elo","version":"Living benchmark"},"leaderboardUrl":"https://huggingface.co/spaces/CityMindDev/LiveHouse-TS","submissionUrl":"https://github.com/zhouziyu02/TS-Live/issues/new?template=community-model.yml"},"curation":{"state":"source-reviewed","reviewedAt":"2026-08-20","sources":["https://arxiv.org/abs/2608.17299","https://arxiv.org/html/2608.17299","https://github.com/zhouziyu02/TS-Live","https://huggingface.co/spaces/CityMindDev/LiveHouse-TS"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.17299","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"evaluationMode":"score_submission","availability":{"paperStatus":"available","githubStatus":"available","hfDatasetStatus":"available","evaluatorStatus":"available","submissionStatus":"available"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"cross-domain"},{"id":"bm_libero-vifo_76763b3b","familyId":"bmf_092e28009a80","name":"LIBERO-VIFO","oneLine":"Evaluates whether vision-language-action models follow authorized visual cues while resisting unauthorized visual influence.","area":"Safety & Trustworthiness","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Embodied manipulation","Visual instruction following","Safety"],"topics":["Multimodal","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.17600","pdf":"https://arxiv.org/pdf/2608.17600","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.17600"},"evidence":{"snippet":"To address these gaps, we introduce LIBERO-VIFO, a benchmark to evaluate both the capability and safety of visual cue following in VLA models.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.17600"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LIBERO-VIFO is a benchmark for visual cue following in vision-language-action models, evaluating capability and safety across eight cue families. However, no public artifacts or official links are provided.","whyItMatters":"If released, it could inform the development of safer VLA systems, but current lack of access limits its practical value.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"299804e0fc9f5979754ed09eb8b0c5057dff36aab38ecfdc75b78d811ac14934"},"motivation":"Visual cues are increasingly adopted to guide robot learning, but whether Vision-Language-Action (VLA) models can reliably follow authorized cues while disregarding unauthorized ones remains unclear.","constructionDetail":"LIBERO-VIFO tests authorized and unauthorized visual influence across robot-manipulation tasks and eight families of visual cues.","detail":{"taskBreakdown":["Text overlays","Geometric symbols","Ghost trajectories","Pictorial demonstrations","Hand gestures","Visual signage","Visual highlights","Machine-readable codes"],"protocol":{"tasks":"1,347 instances from 40 LIBERO tasks and 33 cue variants","primaryMetric":"Full-Chain Accuracy and authorized/unauthorized visual following rates","version":"arXiv v1"}},"curation":{"state":"source-reviewed","reviewedAt":"2026-08-20","sources":["https://arxiv.org/abs/2608.17600","https://arxiv.org/html/2608.17600"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.17600","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"evaluationMode":"score_submission","availability":{"paperStatus":"available","githubStatus":"not_found","hfDatasetStatus":"not_found","evaluatorStatus":"not_found","submissionStatus":"not_found"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_harnessrisk_e838b2b9","familyId":"bmf_171dcd6194e0","name":"HarnessRisk","oneLine":"Evaluates safety failures of model–agent-harness configurations across setup, runtime, persistent state, actions and incident recovery.","area":"Safety & Trustworthiness","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":[],"capabilities":["Agent safety","Prompt-injection resistance","Tool-use safety","Persistent-state safety"],"topics":["Agents","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.17597","pdf":"https://arxiv.org/pdf/2608.17597","project":"https://baiyajing.github.io/harness-risk/","code":"https://github.com/Baiyajing/HarnessRisk","data":"https://huggingface.co/datasets/YajingB/HarnessRisk","hfPaper":"https://huggingface.co/papers/2608.17597"},"evidence":{"snippet":"We present HarnessRisk, a lifecycle oriented benchmark that organizes agent harness safety into six operational phases including Harness Configuration, Capability Extension, Runtime Operation, State Persistence, Action Control, and Incident Recovery.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":9,"hfDailySubmittedAt":"2026-08-19T00:00:00.000Z","githubStars":9,"githubScope":"benchmark_repo","hfDatasetDownloads":37,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2608.17597"},"ranking":{"30d":{"score":66,"rank":37,"coverage":1.0,"confidence":"High","datasetDownloadRank":17,"datasetRankPopulation":20},"90d":{"score":49,"rank":209,"coverage":1.0,"confidence":"High","datasetDownloadRank":46,"datasetRankPopulation":51}},"description":"HarnessRisk evaluates agent harness safety across six lifecycle phases with 128 sandboxed cases. It measures Utility, Attack Success Rate, Persistence, and Detection for each trajectory.","whyItMatters":"Addresses the need for systematic evaluation of agent harness safety across multiple responsibilities, enabling comparison of harness and model combinations and highlighting vulnerability patterns.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fb1a3bd4756a474cb52f9101f8ae4686adcf1d399910348746b8097fbc44dac0"},"motivation":"Large language models are increasingly deployed through agent harnesses that manage tools, extensions, persistent state, permissions, and external actions.","constructionDetail":"HarnessRisk evaluates the safety of complete model–harness configurations across six operational lifecycle phases in fresh sandboxes.","detail":{"taskBreakdown":["Harness configuration","Capability extension","Runtime operation","State persistence","Action control","Incident recovery"],"protocol":{"tasks":"128 sandboxed cases across six lifecycle phases","primaryMetric":"Utility, Attack Success Rate, Persistence and Detection","version":"arXiv v1"}},"curation":{"state":"source-reviewed","reviewedAt":"2026-08-20","sources":["https://arxiv.org/abs/2608.17597","https://arxiv.org/html/2608.17597","https://github.com/Baiyajing/HarnessRisk","https://huggingface.co/datasets/YajingB/HarnessRisk"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.17597","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"evaluationMode":"score_submission","availability":{"paperStatus":"available","githubStatus":"available","hfDatasetStatus":"available","evaluatorStatus":"available","submissionStatus":"not_found"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_cadp-bench_c4c34058","familyId":"bmf_a82a8535ed6e","name":"CADP-Bench","oneLine":"Evaluates MLLMs that reconstruct full academic pages as compilable LaTeX and executable Python from page images.","area":"Multimodal","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":[],"capabilities":["Academic document parsing","Structured document reconstruction","Chart-to-code generation","Multimodal code generation"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.17550","pdf":"https://arxiv.org/pdf/2608.17550","project":"https://github.com/AriKing11/CADP-Bench","code":"https://github.com/AriKing11/CADP-Bench","data":"https://github.com/AriKing11/CADP-Bench/tree/main/data","hfPaper":"https://huggingface.co/papers/2608.17550"},"evidence":{"snippet":"To support this setting, we introduce CADP-Bench, an expert-verified benchmark of full academic pages containing tightly coupled text and multiple SAE types, evaluated through a re-injection compilation protocol.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.17550"},"ranking":{"30d":{"score":34,"rank":126,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":349,"coverage":0.55,"confidence":"Low"}},"description":"CADP-Bench evaluates multimodal LLMs on compilable academic document parsing. It includes expert-verified full academic pages with tightly coupled text and structured elements, assessed through a re-injection compilation protocol.","whyItMatters":"Provides a structured evaluation for structure-aware scientific document parsing, measuring the fidelity of executable reconstructions, which is crucial for machine-readable scientific knowledge.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"85b405ab2d375c2799108825ab66f088802f9ba6930377405a31d41e0d220d9e"},"motivation":"Academic papers are a primary carrier of scientific knowledge, yet most of this knowledge remains locked in PDFs that are optimized for human reading rather than machine use.","constructionDetail":"CADP-Bench asks models to reconstruct complete academic pages as compilable LaTeX and executable Python, then scores both structure and rendered output.","detail":{"taskBreakdown":["Computer Science","Physics","Economics","Quantitative Biology","Statistics"],"protocol":{"tasks":"1,630 full-page samples","primaryMetric":"Pixel Similarity with structural, reading-order and execution metrics","version":"v1"}},"curation":{"state":"source-reviewed","reviewedAt":"2026-08-20","sources":["https://arxiv.org/abs/2608.17550","https://arxiv.org/html/2608.17550v1","https://github.com/AriKing11/CADP-Bench"]},"publication":{"status":"acceptance_claimed","venue":"ACM MM 2026","evidence":"Accepted by ACM MM 2026","evidenceUrl":"https://arxiv.org/abs/2608.17550","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ACM MM 2026","reviewStatus":"accepted","decisionRaw":"Accepted by ACM MM 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.17550","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by ACM MM 2026","level":"author-claim"}]}],"evaluationMode":"score_submission","availability":{"paperStatus":"available","githubStatus":"available","hfDatasetStatus":"sample_only","evaluatorStatus":"available","submissionStatus":"not_found"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_bear-bench_6aff8566","familyId":"bmf_a95253d365a4","name":"BEAR-Bench","oneLine":"Tests English–Russian multimodal reasoning over text-dense business and scientific document pages.","area":"Multimodal","applicationDomains":["Finance & Economics","Science & Research"],"primaryDomain":"Finance & Economics","industrySectors":[],"capabilities":["Professional document reasoning","Multimodal reasoning","OCR-grounded reasoning","Bilingual reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-18","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.17895","pdf":"https://arxiv.org/pdf/2608.17895","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.17895"},"evidence":{"snippet":"To address these limitations, we introduce BEAR-Bench (Bilingual Enterprise and Academic Reasoning), a self-contained, complex English-and-Russian benchmark comprising 1000 human-annotated questions based on text-rich business and scientific documents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.17895"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"BEAR-Bench evaluates multimodal models on bilingual English-Russian text-dense business and scientific documents with 1,000 human-annotated questions.","whyItMatters":"Existing multimodal benchmarks underrepresent bilingual professional document reasoning; BEAR-Bench targets this gap for enterprise and academic use cases.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5ca9fb0fd596963ea1c90fe1ad7ee350039a840a6b14ae7ca9163a5e7e44f457"},"motivation":"While Multimodal Large Language Models (MLLMs) have made significant strides in visual comprehension, their ability to reason about text-dense, professional documents remains incompletely evaluated.","constructionDetail":"BEAR-Bench describes bilingual question answering over business and scientific document pages, but no official benchmark artifact has been released yet.","detail":{"taskBreakdown":["Business","Science"],"protocol":{"tasks":"1,000 document-image questions","primaryMetric":"Accuracy using a semantic-equivalence judge","language":"English and Russian","version":"v1"}},"curation":{"state":"source-reviewed","reviewedAt":"2026-08-20","sources":["https://arxiv.org/abs/2608.17895","https://arxiv.org/html/2608.17895v1"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.17895","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"evaluationMode":"score_submission","availability":{"paperStatus":"available","githubStatus":"not_found","hfDatasetStatus":"not_found","evaluatorStatus":"paper_spec_only","submissionStatus":"not_found"},"displayEligible":false,"capabilityGroups":["Multimodal Perception"],"domainScope":"cross-domain"},{"id":"bm_asi-bench_faf0dd90","familyId":"bmf_3b83443d21ef","name":"ASI-Bench","oneLine":"ASI-Bench evaluates AI systems on 60 project-level scientific research tasks across 11 domains, with four guidance levels B1-B4 measuring autonomous execution and innovation.","area":"Agents & Tool Use","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Research & Development"],"capabilities":["Scientific discovery","Research planning","Autonomous execution"],"topics":["AI Scientist","Agents","Reasoning"],"construction":"Hybrid","annotation":"Expert Generated","readiness":"Runnable","releasedAt":"2026-08-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.17271","pdf":"https://arxiv.org/pdf/2608.17271","project":"https://asibench.apexin.ai","code":"https://github.com/apexin-ai/ASI-Bench","data":"https://huggingface.co/datasets/Apexintelligence-AI/ASI-Bench-seed31415","hfPaper":"https://huggingface.co/papers/2608.17271"},"evidence":{"snippet":"ASI-Bench contains 60 project-level research tasks across 11 scientific domains and progressively reduces methodological guidance to test whether AI can independently select methods, conduct research, and produce verifiable results.","reasonCodes":["named benchmark release","public evaluator","submission path","leaderboard"]},"dataStatus":"primary-source-reviewed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":60,"hfDailySubmittedAt":"2026-08-19T00:00:00.000Z","githubStars":132,"githubScope":"benchmark_repo","hfDatasetDownloads":1615,"hfDatasetLikes":18},"source":{"type":"arxiv","id":"2608.17271"},"ranking":{"30d":{"score":96,"rank":1,"coverage":1.0,"confidence":"High","datasetDownloadRank":3,"datasetRankPopulation":20},"90d":{"score":94,"rank":3,"coverage":1.0,"confidence":"High","datasetDownloadRank":6,"datasetRankPopulation":51}},"description":"ASI-Bench evaluates AI systems on 60 project-level scientific research tasks across 11 domains, with four guidance levels B1-B4 measuring autonomous execution and innovation.","whyItMatters":"It addresses the gap in evaluating AI's independent scientific exploration and execution, revealing current dependence on human guidance for end-to-end research.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4ced4b176bd1572a50abbafd269d41d66f3dce2c7bc8ccc509a202924efd679e"},"motivation":"Evaluate whether AI agents can independently select methods, execute end-to-end research, and produce verifiable scientific results as human methodological guidance is progressively withdrawn.","constructionDetail":"The maintainers distilled more than 1,300 candidate research ideas through five review rounds, over 1,100 review assignments, more than 2,000 task revisions, and over 1,500 sandbox runs.","metrics":[{"name":"Tasks","value":"60","note":"project-level tasks"},{"name":"Coverage","value":"11","note":"scientific domains"},{"name":"Protocol","value":"B1–B4","note":"progressively less methodological guidance"},{"name":"Evaluation","value":"Expert + execution","note":"cross-review, AI-assisted audit, sandbox runs, scorer validation"}],"detail":{"taskBreakdown":["Mathematics","Physics","Chemistry","Biology","Astronomy","Materials science","Earth science","Medicine & biostatistics","Computer science","Robotics","Electrical engineering"],"protocol":{"tasks":60,"conditions":["B1","B2","B3","B4"],"primaryMetric":"Scientific Score","comparisonMetric":"B3 Scientific Score","aggregation":"Macro-average over tasks","runs":"Three independent runs unless marked otherwise","tools":"Paper results reported without external tool access"},"modelCoverage":[{"provider":"OpenAI","models":["GPT-5.5","GPT-5.6 Sol"]},{"provider":"Anthropic","models":["Claude Opus 4.8","Claude Opus 5"]},{"provider":"Moonshot AI","models":["Kimi K2.7","Kimi K3"]},{"provider":"Z.AI","models":["GLM-5.2","GLM-5.3"]},{"provider":"DeepSeek","models":["DeepSeek V4 Flash","DeepSeek V4 Pro"]},{"provider":"MiniMax","models":["MiniMax M3"]},{"provider":"Xiaomi","models":["MiMo V2.5 Pro"]}],"modelCoverageNote":"These systems were evaluated by the benchmark authors; their presence does not by itself mean the model provider adopted or endorsed ASI-Bench.","adoption":{"independentOrganizations":[],"note":"No source-linked independent model-provider adoption has been recorded yet."},"leaderboard":{"primaryMetric":"B3 Scientific Score","bestScore":51.6,"bestSystem":"GPT-5.6 Sol (ultra) · Codex","meanScore":26.62,"evaluatedConfigurations":18,"saturationStatus":"Not saturated","assessment":"Benchmark-author assessment","asOf":"2026-08-18","sourceUrl":"https://arxiv.org/abs/2608.17271"},"leaderboardUrl":"https://asibench.apexin.ai/leaderboard","submissionUrl":"https://asibench.apexin.ai/submit"},"curation":{"state":"source-reviewed","reviewedAt":"2026-08-19","sources":["https://github.com/apexin-ai/ASI-Bench","https://arxiv.org/abs/2608.17271"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.17271","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-20T00:00:00Z"},"releaseDates":{"firstPublicAt":"2026-08-18","paperV1At":"2026-08-18"},"evaluationMode":"score_submission","availability":{"paperStatus":"available","githubStatus":"available","hfDatasetStatus":"available","evaluatorStatus":"available","submissionStatus":"available"},"publishers":[{"name":"Apex Intelligence AI","organizationType":"company-research-lab","sourceUrl":"https://github.com/apexin-ai/ASI-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_safegesture_6fcb400d","familyId":"bmf_4d0bd9e289c6","name":"SafeGesture","oneLine":"SafeGesture evaluates vision-language models on scenario-conditioned safety interpretation of hand gestures, pairing 6 gestures with 8 scenarios for 4,800 items.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.16081","pdf":"https://arxiv.org/pdf/2608.16081","project":null,"code":"https://github.com/The-Responsible-AI-Initiative/SafeGesture","data":null,"hfPaper":"https://huggingface.co/papers/2608.16081"},"evidence":{"snippet":"We introduce SafeGesture, a benchmark that evaluates whether a model can infer scenario-appropriate safety actions from hand gestures.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.16081"},"ranking":{"30d":{"score":34,"rank":127,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":350,"coverage":0.55,"confidence":"Low"}},"description":"SafeGesture evaluates vision-language models on scenario-conditioned safety interpretation of hand gestures, pairing 6 gestures with 8 scenarios for 4,800 items.","whyItMatters":"It exposes a perception-reasoning gap in safety-critical gesture interpretation, providing a reusable test for scenario-dependent decision-making.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"59114b1de9b4fa9805dd83614bcf8dc7c5b9ab52ec9083f2b1aaf1219b3ef0fc"},"motivation":"Open-weight and frontier vision-language models (VLMs) perform well on general image understanding, but their ability to interpret fine-grained hand gestures in safety-critical operational contexts remains largely unexamined.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.16081","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"The Responsible AI Initiative","organizationType":"benchmark-organization","sourceUrl":"https://github.com/The-Responsible-AI-Initiative/SafeGesture","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_reref-3d_34628627","familyId":"bmf_dc0caa547026","name":"ReRef-3D","oneLine":"ReRef-3D benchmarks language-guided 3D scene rearrangement with 33,826 instructions across 998 scenes, evaluating placement validity and relation satisfaction.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.16011","pdf":"https://arxiv.org/pdf/2608.16011","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.16011"},"evidence":{"snippet":"We introduce ReRef-3D, a benchmark for language-guided placement in 3D scenes.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.16011"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ReRef-3D benchmarks language-guided 3D scene rearrangement with 33,826 instructions across 998 scenes, evaluating placement validity and relation satisfaction.","whyItMatters":"It targets spatial reasoning in embodied AI and provides metrics for relation satisfaction and physical validity in rearrangement tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"74e540dbdf23f203f9aad76c7ad4b20439432416f33fe25962c5be48b0b68579"},"motivation":"We introduce ReRef-3D, a benchmark for language-guided placement in 3D scenes.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.16011","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_reconstruction_90ed3fab","familyId":"bmf_d3986ad4c179","name":"Reconstruction","oneLine":"Reconstruction is a blind benchmark for recovering research ideas from pre-publication bibliographies, using a judge model to match hypotheses.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.16645","pdf":"https://arxiv.org/pdf/2608.16645","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.16645"},"evidence":{"snippet":"We introduce Reconstruction, a blind idea-recovery benchmark that withholds the seed paper and all contemporaneous or future literature, and asks models to propose hypotheses that an independent large language model judge matches against the held-out ground-truth idea.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.16645"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Reconstruction is a blind benchmark for recovering research ideas from pre-publication bibliographies, using a judge model to match hypotheses.","whyItMatters":"It tests idea recovery and anti-leakage protocols, but focuses on a specific research question rather than a general comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8bb8e1ee70111f7105a7a72c78af15c69f35906673424414b956b6664aac5074"},"motivation":"Can a language model recover the true research idea of a published paper when given only that paper's pre-publication bibliography?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.16645","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_personashot_741ba748","familyId":"bmf_2db553a9d65f","name":"PersonaShot","oneLine":"PersonaShot benchmarks person-centric narrative continuity in multi-shot video generation with ~1,000 segments and 16 metrics.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.16717","pdf":"https://arxiv.org/pdf/2608.16717","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.16717"},"evidence":{"snippet":"To address these limitations, we introduce PersonaShot, the first person-centric benchmark for narrative continuity in multi-shot video generation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.16717"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PersonaShot benchmarks person-centric narrative continuity in multi-shot video generation with ~1,000 segments and 16 metrics.","whyItMatters":"It addresses the gap in evaluating character coherence across video cuts, including physical and emotional state continuity.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a05db40195b8a4d66c0e57952bf73a94db951d956fb00cd93b728764310a7844"},"motivation":"Video generation is rapidly evolving from single-shot clips to multi-shot narratives, where the human character serves as the core narrative anchor.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.16717","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ov3d-bench_dff2d230","familyId":"bmf_dff0e2aa5738","name":"OV3D-Bench","oneLine":"OV3D-Bench is a diagnostic benchmark for open-vocabulary monocular 3D detection, evaluating localization, semantic robustness, and cross-domain transfer across seven datasets.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.17110","pdf":"https://arxiv.org/pdf/2608.17110","project":null,"code":"https://github.com/mgladkova/ov3d-bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.17110"},"evidence":{"snippet":"To address this, we introduce OV3D-Bench, a diagnostic benchmark that compares open-vocabulary monocular 3D detectors under deployment-realistic conditions across seven indoor and outdoor datasets.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.17110"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OV3D-Bench is a diagnostic benchmark for open-vocabulary monocular 3D detection, evaluating localization, semantic robustness, and cross-domain transfer across seven datasets.","whyItMatters":"It reveals that open-vocabulary semantics is the primary bottleneck, providing a decoupled evaluation axis for detector diagnosis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e51109fc713620a3a78a1d1190632650d677b085e585b55a4111434d1bb235a0"},"motivation":"Open-vocabulary monocular 3D detectors report strong in-domain performance, but each evaluates under a different protocol, several rely on per-image category oracles unavailable at deployment, and all collapse geometry and semantics into a single AP metric.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"OpenSUN3D workshop at ECCV'26","evidence":"Accepted to OpenSUN3D workshop at ECCV'26; benchmark is released on https://github.com/mgladkova/ov3d-bench","evidenceUrl":"https://arxiv.org/abs/2608.17110","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"OpenSUN3D workshop at ECCV'26","reviewStatus":"accepted","decisionRaw":"Accepted to OpenSUN3D workshop at ECCV'26; benchmark is released on https://github.com/mgladkova/ov3d-bench","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.17110","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to OpenSUN3D workshop at ECCV'26; benchmark is released on https://github.com/mgladkova/ov3d-bench","level":"author-claim"}]}],"publishers":[{"name":"University of Ljubljana","organizationType":"academic-lab","sourceUrl":"https://github.com/mgladkova/ov3d-bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_openharmony-bench_a2c42703","familyId":"bmf_ad07fa1e0a63","name":"OpenHarmony Bench","oneLine":"OpenHarmony Bench evaluates coding agents on 153 app-level ArkTS tasks across three input sources, with 242 feature points and device-based verification.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.16022","pdf":"https://arxiv.org/pdf/2608.16022","project":"https://bench.matrix.openharmony.cn/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.16022"},"evidence":{"snippet":"We present OPENHARMONY BENCH, an app-level coding benchmark for evaluating LLM-based coding agents on OpenHarmony ArkTS applications.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.16022"},"ranking":{"30d":{"score":49,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"OpenHarmony Bench evaluates coding agents on 153 app-level ArkTS tasks across three input sources, with 242 feature points and device-based verification.","whyItMatters":"It measures end-to-end app-level correctness, filling the gap between function-level coding benchmarks and real-world app development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a954e61b07650b8569ccbd8e7f1dc95b0b4603681721a0ca94040c81faf4537a"},"motivation":"We present OPENHARMONY BENCH, an app-level coding benchmark for evaluating LLM-based coding agents on OpenHarmony ArkTS applications.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.16022","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"OpenHarmony Community","organizationType":"community","sourceUrl":"https://bench.matrix.openharmony.cn/","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_inspire_4ebdcda6","familyId":"bmf_1cb6508098e9","name":"INSPIRE","oneLine":"INSPIRE evaluates instruction-aware speech retrieval, where natural-language instructions specify relevance criteria including semantic content, speaker identity, speaking style, environmental sounds, and combinations. The benchmark includes a fixed dataset and protocol for evaluating retrieval systems.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.SD"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.16203","pdf":"https://arxiv.org/pdf/2608.16203","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.16203"},"evidence":{"snippet":"We introduce INSPIRE, the first benchmark for instruction-aware speech retrieval, in which natural-language instructions dynamically specify relevance criteria, including semantic content, speaker identity, speaking style, environmental sounds, and their combinations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.16203"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"INSPIRE evaluates instruction-aware speech retrieval, where natural-language instructions specify relevance criteria including semantic content, speaker identity, speaking style, environmental sounds, and combinations. The benchmark includes a fixed dataset and protocol for evaluating retrieval systems.","whyItMatters":"Speech retrieval systems currently rely on fixed similarity matching and cannot adapt to diverse user intents. INSPIRE provides a standardized evaluation to compare methods across different retrieval intents, highlighting gaps in handling both semantic and paralinguistic attributes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2cd70d638e1a860164b2f8e94057b54e84008f39071b41a08f668cb09fda23f0"},"motivation":"Existing speech retrieval systems rely on fixed similarity matching and cannot adapt to diverse user intents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.16203","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Search & Retrieval"],"domainScope":"general"},{"id":"bm_conceptedit-bench_65e0c6a3","familyId":"bmf_1b817470614a","name":"ConceptEdit-Bench","oneLine":"ConceptEdit-Bench is a granular evaluation suite for image editing models. It diagnoses model capabilities across over 1,000 fine-grained edit concepts and real-world scenarios, with a specific protocol for measuring editing performance.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-17","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.16812","pdf":"https://arxiv.org/pdf/2608.16812","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.16812"},"evidence":{"snippet":"Finally, we present ConceptEdit-Bench, a granular evaluation suite designed to diagnose model capabilities across a vast array of real-world scenarios.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.16812"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ConceptEdit-Bench is a granular evaluation suite for image editing models. It diagnoses model capabilities across over 1,000 fine-grained edit concepts and real-world scenarios, with a specific protocol for measuring editing performance.","whyItMatters":"Existing image editing benchmarks lack granularity and coverage. ConceptEdit-Bench provides a more comprehensive diagnostic tool to evaluate models across diverse editing concepts, helping to identify specific strengths and weaknesses.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"48dc7ff5fabb3e49830b50769b6a74ba497af1b8113f35b12debd52edbc93725"},"motivation":"Existing image editing frameworks predominantly follow the training paradigm of text-to-image diffusion models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.16812","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_aerocopilotbench_7dd468f7","familyId":"bmf_cbfa21ffac06","name":"AeroCopilotBench","oneLine":"AeroCopilotBench is a two-tier benchmark for evaluating LLM agents as aviation copilots. Tier-1 uses 1,200 multiple-choice questions for knowledge assessment, while Tier-2 includes 73 procedural tasks in an interactive virtual cockpit environment, with safety-gated evaluation.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.16349","pdf":"https://arxiv.org/pdf/2608.16349","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.16349"},"evidence":{"snippet":"This paper presents the AeroCopilot Operational Environment (ACOE), a reproducible interactive virtual-cockpit test environment, and AeroCopilotBench, a two-tier aviation agent evaluation benchmark.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.16349"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AeroCopilotBench is a two-tier benchmark for evaluating LLM agents as aviation copilots. Tier-1 uses 1,200 multiple-choice questions for knowledge assessment, while Tier-2 includes 73 procedural tasks in an interactive virtual cockpit environment, with safety-gated evaluation.","whyItMatters":"Aviation evaluations often focus on static knowledge and cannot test procedural execution and safety compliance. AeroCopilotBench provides a reproducible interactive environment and safety-gated scoring to assess agents' task completion and trajectory safety.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1d2a2eb64ae7a62f2646b3d67615636af6a1bdce64fcbaa5a187bb30278a28ce"},"motivation":"Large language model (LLM) agents may assist flight crews with complex decisions and task execution, but existing aviation evaluations centered on static knowledge do not support systematic testing of procedural execution and safety compliance in interactive environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.16349","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_wse-bench_2966278a","familyId":"bmf_79b3bfde8025","name":"WSE-bench","oneLine":"WSE-bench evaluates LLM storytelling in open-ended world simulations, assessing three capacities: Generation Coverage (proportion of planned narrative steps), Consistency (canon coherence), and Richness (meaningful development). Uses a process benchmark to compare agent architectures.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-16","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.15654","pdf":"https://arxiv.org/pdf/2608.15654","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.15654"},"evidence":{"snippet":"We introduce WSE-bench, a process benchmark that separately evaluates sustained generation, canonical coherence, and meaningful development in dynamic LLM storytelling.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15654"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"WSE-bench evaluates LLM storytelling in open-ended world simulations, assessing three capacities: Generation Coverage (proportion of planned narrative steps), Consistency (canon coherence), and Richness (meaningful development). Uses a process benchmark to compare agent architectures.","whyItMatters":"Storytelling evaluation has focused on finished stories, but open-ended narratives require sustained generation, coherence, and development. WSE-bench makes these dynamics visible and shows they are distinct capacities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"518516079b768bba91f6c949337ea956a14b141d39a47727c65f70a5793db472"},"motivation":"Large language models can write fluent stories, but open-ended storytelling requires more than local fluency.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15654","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_varm-bench_2f35ad2f","familyId":"bmf_93360e9b8769","name":"VARM-Bench","oneLine":"VARM-Bench evaluates verifiable structured reasoning in Chinese abusive-speech moderation. It uses field-anchored chain-of-thought rationales with six decision fields, and a deterministic protocol assessing field correctness, alignment, output validity, and record errors.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15600","pdf":"https://arxiv.org/pdf/2608.15600","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.15600"},"evidence":{"snippet":"We introduce VARM-Bench, a benchmark for field-anchored chain-of-thought rationales in Chinese abusive-speech moderation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15600"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"VARM-Bench evaluates verifiable structured reasoning in Chinese abusive-speech moderation. It uses field-anchored chain-of-thought rationales with six decision fields, and a deterministic protocol assessing field correctness, alignment, output validity, and record errors.","whyItMatters":"Existing benchmarks support classification but not verifiable reasoning. VARM-Bench provides an auditable protocol for evaluating moderation rationales, revealing that strong label performance can conceal errors in complete records.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0e7cbac4d6c2f3731a5ce2ebf627f02213295f0839649ceaf5d7919c7bc8b219"},"motivation":"The widespread circulation of abusive online content has increased the need for reliable moderation of Chinese social-media text.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15600","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_rigidbench_f95c26d5","familyId":"bmf_c9cf6af36be4","name":"RigidBench","oneLine":"RigidBench evaluates rigid-body physics in video generation models using a simulator-grounded benchmark with 100 examples and five tasks. It provides ten measurements covering motion, geometry, identity, background stability, and appearance, with per-frame data.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15555","pdf":"https://arxiv.org/pdf/2608.15555","project":"https://doi.org/10.5281/zenodo.21649156","code":"https://github.com/swarnim-j/RigidBench","data":null,"hfPaper":"https://huggingface.co/papers/2608.15555"},"evidence":{"snippet":"We introduce RigidBench, a simulator-grounded benchmark that compares a generated continuation with a reference rollout from the same initial frame and motion description.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15555"},"ranking":{"30d":{"score":50,"rank":70,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":279,"coverage":0.55,"confidence":"Low"}},"description":"RigidBench evaluates rigid-body physics in video generation models using a simulator-grounded benchmark with 100 examples and five tasks. It provides ten measurements covering motion, geometry, identity, background stability, and appearance, with per-frame data.","whyItMatters":"Video generation metrics often mix independent errors. RigidBench separates these aspects and shows rankings depend on what is measured, providing a more detailed evaluation for physics fidelity.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"687a65c5c85f54877092b3b4c0c0049c51bfdb67e9bad140c78ca8974a31e4f6"},"motivation":"Video models are increasingly used to predict what happens next in a scene, yet the metrics commonly used to compare their outputs say little about whether the predicted objects move correctly.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15555","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"RigidBench contributors","organizationType":"academic-lab","sourceUrl":"https://github.com/swarnim-j/RigidBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_plsqlbench_adc42146","familyId":"bmf_5ba5b1f87407","name":"PLSQLBench","oneLine":"PLSQLBench evaluates LLMs' ability to write executable PL/SQL programs through execution-based tests. It contains 2,865 instances including single-turn and multi-turn tasks, covering schema-grounded and procedural problems.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation","Factuality"],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15931","pdf":"https://arxiv.org/pdf/2608.15931","project":null,"code":"https://github.com/oracle-samples/plsqlbench","data":null,"hfPaper":"https://huggingface.co/papers/2608.15931"},"evidence":{"snippet":"We present PLSQLBench, to our knowledge the first benchmark for evaluating whether LLMs can write executable PL/SQL programs, with correctness measured through execution-based tests.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15931"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PLSQLBench evaluates LLMs' ability to write executable PL/SQL programs through execution-based tests. It contains 2,865 instances including single-turn and multi-turn tasks, covering schema-grounded and procedural problems.","whyItMatters":"Existing evaluations target general code generation or declarative text-to-SQL, leaving procedural database programming underexplored. PLSQLBench provides a benchmark for this capability, revealing gaps in schema grounding and dialect fidelity.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4e3af412c98d67b6c8c4c851ca509ffe6046362ee48cb7631a961e15fed3543f"},"motivation":"We present PLSQLBench, to our knowledge the first benchmark for evaluating whether LLMs can write executable PL/SQL programs, with correctness measured through execution-based tests.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15931","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Oracle Samples","organizationType":"company-research-lab","sourceUrl":"https://github.com/oracle-samples/plsqlbench","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_l3cube-indicquest-v2_c9db7d34","familyId":"bmf_da7eeb9063e2","name":"L3Cube-IndicQuest v2","oneLine":"L3Cube-IndicQuest v2 is a multilingual QA benchmark for evaluating India-specific factual knowledge of LLMs. It contains 3,471 curriculum-grounded English QA pairs translated into 19 Indic languages, totaling 69,420 pairs across 20 languages.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15535","pdf":"https://arxiv.org/pdf/2608.15535","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.15535"},"evidence":{"snippet":"We present L3Cube-IndicQuest v2, a large-scale gold-standard multilingual question-answering benchmark for evaluating the India-specific factual knowledge of Large Language Models (LLMs).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15535"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"L3Cube-IndicQuest v2 is a multilingual QA benchmark for evaluating India-specific factual knowledge of LLMs. It contains 3,471 curriculum-grounded English QA pairs translated into 19 Indic languages, totaling 69,420 pairs across 20 languages.","whyItMatters":"There is a need for benchmarks evaluating factual knowledge in Indic languages. This benchmark provides a large-scale gold-standard dataset with multiple evaluation protocols, showing consistent rankings across judges.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dbe6a5bed92d7a7ac9db02690ec25c0a8a5bf50fa15bf132f0fc67ad5809f265"},"motivation":"We present L3Cube-IndicQuest v2, a large-scale gold-standard multilingual question-answering benchmark for evaluating the India-specific factual knowledge of Large Language Models (LLMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15535","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_crossview_b52b71d9","familyId":"bmf_fb565242d442","name":"CrossView","oneLine":"CrossView is a multi-camera video question-answering benchmark spanning autonomous driving, security surveillance, egocentric/exocentric video, and robotics. It evaluates vision-language models on tasks that require joint reasoning across multiple simultaneous camera views, including resolving occlusions and integrating evidence across perspectives.","area":"Multimodal","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Automotive"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15539","pdf":"https://arxiv.org/pdf/2608.15539","project":"https://utaustin-swarmlab.github.io/CrossView","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.15539"},"evidence":{"snippet":"We introduce CrossView, a multi-camera video question-answering benchmark spanning autonomous driving, security surveillance, egocentric/exocentric video, and robotics.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15539"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CrossView is a multi-camera video question-answering benchmark spanning autonomous driving, security surveillance, egocentric/exocentric video, and robotics. It evaluates vision-language models on tasks that require joint reasoning across multiple simultaneous camera views, including resolving occlusions and integrating evidence across perspectives.","whyItMatters":"Existing video benchmarks focus on single-camera settings, leaving multi-camera reasoning unmeasured. CrossView provides a standardized evaluation for a capability critical to real-world applications like autonomous vehicles and surveillance, enabling comparison of models on tasks that scale with viewpoint number and require cross-view integration.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"83073cc8b29d61af4121191ab872902c2af21700d4c3fb8a7416bc1431cf7279"},"motivation":"Video understanding benchmarks have long centered on single-camera settings, where modern multi-modal language models achieve strong performance across image and video tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15539","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"UT Austin Swarm Lab","organizationType":"academic-lab","sourceUrl":"https://utaustin-swarmlab.github.io/CrossView","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_wesce_79f8889e","familyId":"bmf_8411e00796ba","name":"WeSCE","oneLine":"WeSCE is a benchmark for quantifying security drift in LLM-driven code editing. It consists of 400 executable programs derived from real-world code, covering feature addition, removal, bug fixing, and refactoring. The benchmark proposes a continuous risk representation and drift measures that capture changes in overall risk, worst-case severity, and vulnerability distribution.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15092","pdf":"https://arxiv.org/pdf/2608.15092","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.15092"},"evidence":{"snippet":"In this work, we introduce WeSCE, a benchmark for quantifying security drift in code editing under weak-security constraints, where tasks specify only functional objectives without explicit security requirements.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15092"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"WeSCE is a benchmark for quantifying security drift in LLM-driven code editing. It consists of 400 executable programs derived from real-world code, covering feature addition, removal, bug fixing, and refactoring. The benchmark proposes a continuous risk representation and drift measures that capture changes in overall risk, worst-case severity, and vulnerability distribution.","whyItMatters":"Code editing with weak-security constraints can introduce vulnerabilities, but existing evaluations lack a systematic measure. WeSCE offers a standardized way to quantify security drift, potentially aiding in selecting models and prompts that minimize security risks during code modifications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5e413f7937e8cce6dd18cbbee3a2fa72ece0340558e02f94ac0d2ffdbc41c6d1"},"motivation":"In this work, we introduce WeSCE, a benchmark for quantifying security drift in code editing under weak-security constraints, where tasks specify only functional objectives without explicit security requirements.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15092","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_longrca-bench_4c5e770a","familyId":"bmf_a00231fd543d","name":"LongRCA Bench","oneLine":"LongRCA Bench is a benchmark for diagnosing responsible roles and root causes in long-horizon agent failures. It comprises 1,140 failed trajectories across five domains, with human labels for the responsible role and the earliest decisive root-cause step. Evaluation focuses on responsible-role accuracy and exact root-step accuracy.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15242","pdf":"https://arxiv.org/pdf/2608.15242","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.15242"},"evidence":{"snippet":"We introduce LongRCA Bench, comprising 1,140 failed trajectories across five domains without injected errors.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15242"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"LongRCA Bench is a benchmark for diagnosing responsible roles and root causes in long-horizon agent failures. It comprises 1,140 failed trajectories across five domains, with human labels for the responsible role and the earliest decisive root-cause step. Evaluation focuses on responsible-role accuracy and exact root-step accuracy.","whyItMatters":"Long-horizon agent failures are difficult to debug, and outcome-level metrics obscure where errors occur. LongRCA Bench provides a standardized testbed for failure attribution, enabling comparison of methods that localize root causes and assign responsibility, which is crucial for improving agent reliability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"18a8bafb2f851f51d696b408dd44815dc1f03e4f22efa0eeb56cff0438e9cde5"},"motivation":"When a long-horizon agent execution fails, outcome-level evaluation reveals the unsuccessful result but not where the decisive error entered the trajectory.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15242","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LongRCA Bench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.15242","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_longdocbench_5cb5bcf9","familyId":"bmf_25cf6240e611","name":"LongDocBench","oneLine":"LongDocBench is a benchmark for Table-of-Contents Hierarchy Recovery and Contextual Relationship Recovery in long documents. It includes 85 real-world documents (financial reports, textbooks, academic papers) spanning 2,582 pages, with human-verified annotations for 3,937 heading nodes and 3,258 contextual relationships.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15064","pdf":"https://arxiv.org/pdf/2608.15064","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.15064"},"evidence":{"snippet":"To benchmark these two tasks, we introduce \\textsc{LongDocBench}, comprising 85 real-world financial reports, textbooks, and academic papers spanning 2,582 pages, with up to 105 pages per document.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15064"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LongDocBench is a benchmark for Table-of-Contents Hierarchy Recovery and Contextual Relationship Recovery in long documents. It includes 85 real-world documents (financial reports, textbooks, academic papers) spanning 2,582 pages, with human-verified annotations for 3,937 heading nodes and 3,258 contextual relationships.","whyItMatters":"Existing document parsing benchmarks focus on page-level tasks, leaving document-level structure recovery unevaluated. LongDocBench provides a standardized evaluation for tasks that are critical for understanding long documents, enabling comparison of parsers on hierarchy and relationship recovery.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bf1f43a4a7ff195e80e0cbf6f97d300f9114d2118aee8cde95428800039282d6"},"motivation":"Parsing visual documents into machine-readable representations is fundamental to document intelligence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15064","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LongDocBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.15064","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_floodreasonbench_bd08edc8","familyId":"bmf_c65f287d8be1","name":"FloodReasonBench","oneLine":"FloodReasonBench is a benchmark for vision-language model reasoning segmentation in flood response scenarios. It introduces FloodResponseSeg, a flood-specific dataset, and evaluates pipelines under lightweight visual encoding, split inference, and compressed representations. It also measures accuracy, latency, energy, and communication tradeoffs on an embedded platform.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15410","pdf":"https://arxiv.org/pdf/2608.15410","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.15410"},"evidence":{"snippet":"We present FloodReasonBench, a benchmark for VLM reasoning segmentation for embodied flood response at the edge.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15410"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FloodReasonBench is a benchmark for vision-language model reasoning segmentation in flood response scenarios. It introduces FloodResponseSeg, a flood-specific dataset, and evaluates pipelines under lightweight visual encoding, split inference, and compressed representations. It also measures accuracy, latency, energy, and communication tradeoffs on an embedded platform.","whyItMatters":"Reasoning segmentation for flood response has domain-specific constraints, and this benchmark characterizes model performance and system-level tradeoffs at the edge, which could inform deployment decisions for resource-constrained platforms.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b194550bd74d53b69aa27092dc5f9186a08edb06b78004764fcfa55eda96ab14"},"motivation":"Reasoning segmentation enables vision-language models (VLMs) to translate mission-relevant language requests into pixel-level visual grounding, offering a natural perception interface for embodied agents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15410","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_finfraudbench_fc16410f","familyId":"bmf_8a9ae6e0eb11","name":"FinFraudBench","oneLine":"FinFraudBench is a heterogeneous graph benchmark for financial fraud detection. It contains two datasets (CreditCard-Fraud and BankTrans-Fraud) with up to 8.99M nodes and 89.23M directed typed edges, preserving six financial entity types and fourteen edge types. The evaluation protocol covers ranking and imbalance-sensitive classification metrics.","area":"Vision & 3D","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15177","pdf":"https://arxiv.org/pdf/2608.15177","project":"https://anonymous.4open.science/r/FinFraudBench-B002","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.15177"},"evidence":{"snippet":"To address these gaps, we present FinFraudBench, a heterogeneous graph benchmark for financial fraud detection.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15177"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FinFraudBench is a heterogeneous graph benchmark for financial fraud detection. It contains two datasets (CreditCard-Fraud and BankTrans-Fraud) with up to 8.99M nodes and 89.23M directed typed edges, preserving six financial entity types and fourteen edge types. The evaluation protocol covers ranking and imbalance-sensitive classification metrics.","whyItMatters":"Existing graph-based fraud detection benchmarks often oversimplify financial systems and lack realistic conditions. FinFraudBench provides large-scale heterogeneous graphs with natural fraud rates, enabling more realistic evaluation of fraud detection methods and comparison across models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2c048e1de37db1a521f9da04691dc7aa1f62ad85fa6d9d6255a3cf86e4592c1c"},"motivation":"The increasing complexity of digital financial systems has reshaped financial fraud detection from isolated transaction classification into relational risk reasoning over interconnected financial entities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15177","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FinFraudBench Team","organizationType":"academic-lab","sourceUrl":"https://anonymous.4open.science/r/FinFraudBench-B002","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_cbx-bench_5fdbe064","familyId":"bmf_825405358cac","name":"CBX-Bench","oneLine":"CBX-Bench is a benchmark for quantitatively measuring the quality of Concept Bottleneck Model (CBM) explanations. It uses a council of five open-weight multimodal LLMs to score explanations given an image and class, validated against human preferences. The benchmark maintains a leaderboard for CBM explanation quality.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15404","pdf":"https://arxiv.org/pdf/2608.15404","project":null,"code":"https://github.com/meric-karadag/cbx-bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.15404"},"evidence":{"snippet":"Building on this validated council, we introduce CBX-Bench, a public benchmark and leaderboard: authors of new CBMs can submit their model's explanations, and CBX-Bench scores them with the council and maintains dataset-level rankings of explanation quality.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15404"},"ranking":{"30d":{"score":34,"rank":128,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":351,"coverage":0.55,"confidence":"Low"}},"description":"CBX-Bench is a benchmark for quantitatively measuring the quality of Concept Bottleneck Model (CBM) explanations. It uses a council of five open-weight multimodal LLMs to score explanations given an image and class, validated against human preferences. The benchmark maintains a leaderboard for CBM explanation quality.","whyItMatters":"CBM interpretability is often evaluated by downstream accuracy, lacking quantitative measures of explanation quality. CBX-Bench offers a human-aligned, scalable evaluation that does not require concept ground truth, enabling comparison of explanation quality across different CBMs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"effde83cc5955e30102b6fbc7361e7b38ab5ce99cfa0bc1f196823f7771a4991"},"motivation":"Concept Bottleneck Models (CBMs) are designed to make visual classification interpretable by expressing predictions through human-understandable concepts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15404","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"CBX-Bench Team","organizationType":"academic-lab","sourceUrl":"https://github.com/meric-karadag/cbx-bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_agentsysbench_213ca6e5","familyId":"bmf_ff8fb088901b","name":"AgentSysBench","oneLine":"AgentSysBench is a benchmark suite and measurement toolkit for characterizing agentic workloads on LLM serving systems. It includes ten representative agentic applications and unified instrumentation, identifying six properties that distinguish agentic workloads from conventional LLM inference.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.OS"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.15127","pdf":"https://arxiv.org/pdf/2608.15127","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.15127"},"evidence":{"snippet":"We present AgentSysBench, a benchmark suite and measurement toolkit with ten representative agentic applications and unified systems-level instrumentation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.15127"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AgentSysBench is a benchmark suite and measurement toolkit for characterizing agentic workloads on LLM serving systems. It includes ten representative agentic applications and unified instrumentation, identifying six properties that distinguish agentic workloads from conventional LLM inference.","whyItMatters":"Serving systems are designed for conventional LLM inference and may not handle agentic workloads efficiently. AgentSysBench provides a measurement-based characterization that could guide system design for agentic applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"669ded87be577731c12b75ef463ae18277f70618c04fa3be18502ca2e58a8632"},"motivation":"Agentic applications are shifting AI serving from isolated model inference to long-running workloads in which LLMs coordinate tools, environments, and persistent state.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.15127","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_timesage-ev_e6de9c4a","familyId":"bmf_8aaa3b125367","name":"TimeSage-EV","oneLine":"TimeSage-EV evaluates LLM agents on time series analysis tasks in evolving environments, using 60 institutional scenarios across 6 domains with 1,485 scenario-period QA pairs. Agents receive data and reports, with withheld target releases as ground truth, assessing state identification, data summarization, and outlook reasoning.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.14270","pdf":"https://arxiv.org/pdf/2608.14270","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.14270"},"evidence":{"snippet":"We introduce TimeSage-EV, a live benchmark for agentic time series analysis in evolving environments.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14270"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TimeSage-EV evaluates LLM agents on time series analysis tasks in evolving environments, using 60 institutional scenarios across 6 domains with 1,485 scenario-period QA pairs. Agents receive data and reports, with withheld target releases as ground truth, assessing state identification, data summarization, and outlook reasoning.","whyItMatters":"Existing time series QA benchmarks rely on fixed snapshots, but real-world data is released periodically, affecting conclusions. TimeSage-EV fills this gap by evaluating agentic temporal validity and cutoff-aware evidence use, providing a decision-value for deploying LLM agents in high-stakes domains where data updates matter.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"470e405966f82e0b28501a51e4b77551a24e44cf36ec8afb7d78bfb34e129b83"},"motivation":"Time series analysis in high-stakes domains relies on recurring data releases, where new observations can alter the evidence base and the validity of later conclusions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14270","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TimeSage-EV Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.14270","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_scope-bench_aaf50d3b","familyId":"bmf_7f2b8edd4cc8","name":"SCOPE-Bench","oneLine":"SCOPE-Bench evaluates short-video recommendation systems by quantifying content depth using the Content Depth Score (CDS), a seven-level scale based on cognitive psychology. It provides CDS annotations for 150K videos from an open-source dataset, enabling systematic assessment of recommenders from a cognitive-content perspective.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.13990","pdf":"https://arxiv.org/pdf/2608.13990","project":"https://liweidengdavid.github.io/SCOPE-Bench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.13990"},"evidence":{"snippet":"As an initial step toward this vision, we present \\textbf{SCOPE-Bench}, the first benchmark for content-depth evaluation in short-video recommendation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13990"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SCOPE-Bench evaluates short-video recommendation systems by quantifying content depth using the Content Depth Score (CDS), a seven-level scale based on cognitive psychology. It provides CDS annotations for 150K videos from an open-source dataset, enabling systematic assessment of recommenders from a cognitive-content perspective.","whyItMatters":"Existing short-video recommenders optimize for engagement, often favoring shallow content. SCOPE-Bench addresses the lack of benchmarks for content depth in recommendation, allowing evaluation of algorithms on their ability to recommend cognitively deep content, which has implications for user well-being and long-term engagement.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d9196b6b479a5eaaf7bffcd39868056996e167973770566ffa342a1e1b10834f"},"motivation":"Driven by the attention economy, short-video Recommender Systems (RSs) are primarily optimized to maximize user engagement by promoting videos that capture attention within seconds.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13990","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_reflexbench_c4d78abf","familyId":"bmf_0c3d9006dea6","name":"ReflexBench","oneLine":"Evaluates vision-language-action models on reaction-critical manipulation across six dynamic tasks, with configurable latency under synchronous and asynchronous inference in a simulated environment.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics","Multimodal"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.14379","pdf":"https://arxiv.org/pdf/2608.14379","project":"https://reflexvla.github.io","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.14379"},"evidence":{"snippet":"To address this gap, we present ReflexBench, a benchmark for reaction-critical manipulation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14379"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates vision-language-action models on reaction-critical manipulation across six dynamic tasks, with configurable latency under synchronous and asynchronous inference in a simulated environment.","whyItMatters":"Addresses the lack of benchmarks for dynamic interaction scenarios in robotic manipulation, providing a standardized way to assess model performance under reaction-critical conditions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"bdc4d99df748d1c2299dff99899805226400fd6c05aa0524ede64b68926d761e"},"motivation":"Vision-Language-Action (VLA) models have recently achieved promising performance in robotic manipulation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14379","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ReflexVLA Project","organizationType":"academic-lab","sourceUrl":"https://reflexvla.github.io","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_ra-bench_8deff463","familyId":"bmf_d6d2987131f5","name":"RA-Bench","oneLine":"RA-Bench evaluates AI-generated video detection using real crisis-event videos as anchors. It includes 17,886 videos: 1,830 real anchors across 10 social-risk categories and 16,056 generated clips from nine generators. Detection is scored per source via AUC and TPR@5%FPR, with separate tracks for human-deceptive and dissemination-processed videos.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.14391","pdf":"https://arxiv.org/pdf/2608.14391","project":null,"code":"https://github.com/24029100313/RA-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.14391"},"evidence":{"snippet":"To address this gap, we introduce RA-Bench, a benchmark for AI-generated video detection that uses Real videos as Anchors.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":277,"hfDailySubmittedAt":"2026-08-17T00:00:00.000Z","githubStars":61,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14391"},"ranking":{"30d":{"score":90,"rank":3,"coverage":0.85,"confidence":"High"},"90d":{"score":80,"rank":23,"coverage":0.7,"confidence":"Medium"}},"description":"RA-Bench evaluates AI-generated video detection using real crisis-event videos as anchors. It includes 17,886 videos: 1,830 real anchors across 10 social-risk categories and 16,056 generated clips from nine generators. Detection is scored per source via AUC and TPR@5%FPR, with separate tracks for human-deceptive and dissemination-processed videos.","whyItMatters":"Current detectors lack evidence on realistic AI-generated crisis videos. RA-Bench provides a matched real-generated evaluation to assess detector generalization, human perception, and robustness to social dissemination, supporting practical decisions on misinformation defense.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"856c736b7dc8596472120f97a8e1b966a697ea59dc18f7f66978cc093746679d"},"motivation":"Recent video generators can fabricate realistic depictions of wars, disasters, public emergencies, and other real-world crises, creating substantial risks of misinformation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14391","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_principle-bench_119ba0f5","familyId":"bmf_dd484486344c","name":"Principle-Bench","oneLine":"Principle-Bench contains 168 cryptoasset financial-promotion scenarios mapped to two UK FCA principles, with paraphrase, adversarial keyword-stuffing, and boundary perturbations for evaluating LLM-as-judge on accuracy, paraphrase robustness, adversarial robustness, and calibration.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.CR"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-14","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.14329","pdf":"https://arxiv.org/pdf/2608.14329","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.14329"},"evidence":{"snippet":"We release Principle-Bench, 168 cryptoasset financial-promotion scenarios mapped to two UK FCA principles, with paraphrase, adversarial keyword-stuffing, and boundary perturbations authored under a pre-registered rubric; the first benchmark covering all four axes for principle-based regulation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14329"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Principle-Bench contains 168 cryptoasset financial-promotion scenarios mapped to two UK FCA principles, with paraphrase, adversarial keyword-stuffing, and boundary perturbations for evaluating LLM-as-judge on accuracy, paraphrase robustness, adversarial robustness, and calibration.","whyItMatters":"Addresses the evaluation gap for LLM-as-judge in principle-based regulation, where standards are not binary. Provides a multi-axis assessment to inform deployment decisions, as no single method dominates all axes and adversarial inputs can significantly degrade performance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7a8be2a0d792a4609acdb74afa5db195fc0626601ae2ac368b3a535b30201848"},"motivation":"Principle-based regulation, with evaluative standards such as \"fair, clear, and not misleading\" or \"deliver good outcomes\", cannot be reduced to binary predicates, and LLM-as-judge is increasingly used as the substitute.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"KDD 2026 Workshop on Secure and Trustworthy Large Language Models (SeT-LLM), poster","evidence":"7 pages, 3 figures. Accepted at the KDD 2026 Workshop on Secure and Trustworthy Large Language Models (SeT-LLM), poster","evidenceUrl":"https://arxiv.org/abs/2608.14329","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"KDD 2026 Workshop on Secure and Trustworthy Large Language Models (SeT-LLM), poster","reviewStatus":"accepted","decisionRaw":"7 pages, 3 figures. Accepted at the KDD 2026 Workshop on Secure and Trustworthy Large Language Models (SeT-LLM), poster","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.14329","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"7 pages, 3 figures. Accepted at the KDD 2026 Workshop on Secure and Trustworthy Large Language Models (SeT-LLM), poster","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_pace-bench_0530dc3f","familyId":"bmf_91838779e882","name":"PACE-Bench","oneLine":"PACE-Bench is a simulator-grounded benchmark with 144 source-to-target adaptation pairs across six physics domains. Each pair presents a code-driven design that succeeds in a source environment but fails in a mutated target environment, and agents must iteratively adapt the design using diagnostic sandbox feedback within a limited attempt budget.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Self-Evolution","Agents"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-14","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.14441","pdf":"https://arxiv.org/pdf/2608.14441","project":null,"code":"https://github.com/thunlp/PACE-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.14441"},"evidence":{"snippet":"To address this gap, we introduce PACE-Bench (Physics Adaptation via Code Evolution), a simulator-grounded benchmark of 144 source-to-target adaptation pairs across six physics domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":28,"hfDailySubmittedAt":"2026-08-18T00:00:00.000Z","githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14441"},"ranking":{"30d":{"score":73,"rank":23,"coverage":0.85,"confidence":"High"},"90d":{"score":57,"rank":155,"coverage":0.7,"confidence":"Medium"}},"description":"PACE-Bench is a simulator-grounded benchmark with 144 source-to-target adaptation pairs across six physics domains. Each pair presents a code-driven design that succeeds in a source environment but fails in a mutated target environment, and agents must iteratively adapt the design using diagnostic sandbox feedback within a limited attempt budget.","whyItMatters":"Existing self-evolving agent evaluations assume fixed execution conditions and do not test recovery after environmental shifts. PACE-Bench provides a repeatable protocol to assess an agent's ability to adapt to changing physics, offering insight into the reliability of different self-evolving methods.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"190109402f0e67b5f2d47cd58eab25ad418d453350fc095b2733be22d22423bf"},"motivation":"Self-evolving agents improve future behavior from interaction experience, yet existing evaluations typically optimize under fixed execution conditions and do not test recovery after those conditions change.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14441","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"THUNLP","organizationType":"academic-lab","sourceUrl":"https://github.com/thunlp/PACE-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_medclawbench_bcd0f6e4","familyId":"bmf_56afb80b1bd4","name":"MedClawBench","oneLine":"MedClawBench evaluates long-horizon temporal reasoning in surgical videos through 1,123 doctor-grounded questions over self-built long neurosurgery recordings and a public lecture-video test split, with fixed evaluation dimensions for comparison.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Agents","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-14","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.14015","pdf":"https://arxiv.org/pdf/2608.14015","project":"https://fyycs.github.io/medclaw/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.14015"},"evidence":{"snippet":"To evaluate this agent, we introduce MedClawBench, a de-leaked, doctor-grounded benchmark of 1,123 questions over self-built long neurosurgery recordings and a held-out public lecture-video test split.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14015"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MedClawBench evaluates long-horizon temporal reasoning in surgical videos through 1,123 doctor-grounded questions over self-built long neurosurgery recordings and a public lecture-video test split, with fixed evaluation dimensions for comparison.","whyItMatters":"Existing VLM benchmarks fail to capture temporal dependencies in long surgical videos. MedClawBench provides a reproducible, doctor-grounded dataset to assess video reasoning capabilities beyond short clips, aiding progress in surgical AI systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7156f19c8562b3031ef6b9a38928e198f2688ff963fa2e2732adb09d9170c6a8"},"motivation":"Understanding tens-of-minutes surgical videos requires long-horizon temporal reasoning, answering what happens before, after, or across stages of a procedure by grounding the question in visual evidence spread across time.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14015","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MedClaw team","organizationType":"academic-lab","sourceUrl":"https://fyycs.github.io/medclaw/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_gbu-palm_d39e8a5c","familyId":"bmf_fcc9a8d5350b","name":"GBU-Palm","oneLine":"GBU-Palm is a large-scale multimodal video dataset for palm presentation attack detection, containing 21,326 videos from 105 subjects across six acquisition environments, including bona fide, Print, and Replay attacks, with 6,310 synchronized RGB-NIR samples. It evaluates video architectures under environment-matched and held-out-environment protocols, using metrics such as true accept, true reject, false accept, and false reject rates.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.14389","pdf":"https://arxiv.org/pdf/2608.14389","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.14389"},"evidence":{"snippet":"We present GBU-Palm, a large-scale multimodal video dataset and benchmark containing 21,326 videos from 105 subjects and 210 palms across six acquisition environments, including bona fide, Print, and Replay presentations, with 6,310 synchronized RGB-NIR samples.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14389"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GBU-Palm is a large-scale multimodal video dataset for palm presentation attack detection, containing 21,326 videos from 105 subjects across six acquisition environments, including bona fide, Print, and Replay attacks, with 6,310 synchronized RGB-NIR samples. It evaluates video architectures under environment-matched and held-out-environment protocols, using metrics such as true accept, true reject, false accept, and false reject rates.","whyItMatters":"Existing palm PAD datasets are limited by static imagery and restricted conditions, hindering systematic evaluation. GBU-Palm provides a unified benchmark to assess robustness across environments and modalities, helping practitioners choose architectures that generalize under environmental shift.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"aed987b576ae0994b262bdf02a421565e78b2615fb0504396c60b522b16f5f96"},"motivation":"Existing palm presentation attack detection (PAD) datasets are often limited by static imagery, restricted acquisition conditions, or insufficient multimodal video data, hindering systematic evaluation across environments, modalities, and attack types.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14389","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GBU-Palm Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.14389","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_flatlab_52af74dd","familyId":"bmf_694513e572ba","name":"FlatLab","oneLine":"FlatLab is a simulation-based benchmark for robotic manipulation of flat objects, providing physical simulation, data collection, and standardized task definitions and evaluation protocols.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.14049","pdf":"https://arxiv.org/pdf/2608.14049","project":"https://flatlab-web.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.14049"},"evidence":{"snippet":"To enable systematic evaluation, we introduce FlatLab, a comprehensive simulation benchmark for robotic flat object manipulation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14049"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FlatLab is a simulation-based benchmark for robotic manipulation of flat objects, providing physical simulation, data collection, and standardized task definitions and evaluation protocols.","whyItMatters":"It addresses the lack of standardized evaluation for flat object manipulation, a challenging domain due to ungraspable configurations and varied materials.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"354beeaa0dd82828229136b5c06de4ac752c63d6acda8a46c70ecca9456ae104"},"motivation":"Robotic manipulation of flat objects is challenging due to the ungraspable configurations and strong variations in object geometry and material.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026","evidence":"This paper is accepted to ICML 2026","evidenceUrl":"https://arxiv.org/abs/2608.14049","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026","reviewStatus":"accepted","decisionRaw":"This paper is accepted to ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.14049","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"This paper is accepted to ICML 2026","level":"author-claim"}]}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_cpi-bench_2cecc896","familyId":"bmf_6f391349d28c","name":"CPI-Bench","oneLine":"CPI-Bench evaluates image editing models across general, practical, and intelligent tasks with VLM-as-Judge scoring, including multi-image and reasoning-based editing.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.14546","pdf":"https://arxiv.org/pdf/2608.14546","project":null,"code":"https://github.com/zqyzzz/CPI-benchmark","data":null,"hfPaper":"https://huggingface.co/papers/2608.14546"},"evidence":{"snippet":"To address these limitations, we propose CPI-Bench, a Comprehensive, Practical and Intelligent benchmark for real-world image editing.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":16,"hfDailySubmittedAt":"2026-08-17T00:00:00.000Z","githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14546"},"ranking":{"30d":{"score":72,"rank":28,"coverage":0.85,"confidence":"High"},"90d":{"score":58,"rank":145,"coverage":0.7,"confidence":"Medium"}},"description":"CPI-Bench evaluates image editing models across general, practical, and intelligent tasks with VLM-as-Judge scoring, including multi-image and reasoning-based editing.","whyItMatters":"It provides a comprehensive benchmark that captures real-world deployment scenarios and reasoning demands, with scoring aligned to human preferences for reliable model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0604b1a0b8824409d44a66b794d09d481d64bc8fb557dd3a42663a95ffe1b6e3"},"motivation":"With the rapid advancement of image editing models and their widespread application across various domains, there is an increasingly urgent need to deploy these model capabilities directly into real-world scenarios.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14546","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TaobaoTmall-AlgorithmProducts","organizationType":"company-research-lab","sourceUrl":"https://github.com/zqyzzz/CPI-benchmark","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_anchorbench_ca7f11f6","familyId":"bmf_f6ee7b787418","name":"AnchorBench","oneLine":"AnchorBench evaluates anchoring effects in LLMs across multiple pathways and anchor relevance, measuring answer shifts through controlled prompts.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.14320","pdf":"https://arxiv.org/pdf/2608.14320","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.14320"},"evidence":{"snippet":"We introduce AnchorBench, a benchmark for the anchoring effect in LLMs that evaluates multiple anchor pathways under an explicit anchor relevance axis.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14320"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AnchorBench evaluates anchoring effects in LLMs across multiple pathways and anchor relevance, measuring answer shifts through controlled prompts.","whyItMatters":"It addresses gaps in prior anchoring benchmarks by distinguishing irrelevant and plausible anchors and evaluating pathway dependence.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8619b854c7529cc19e7ea21738ad0fb45e00f264c5d0cc101efcdd4d2a22adb2"},"motivation":"The anchoring effect is a cognitive bias in which an initial reference value shifts a later judgment toward itself.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14320","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_vero_3ed8d572","familyId":"bmf_5cca655ac072","name":"Vero","oneLine":"Vero evaluates repository-level verified code generation in Lean 4, with 43 multi-module instances, formal specifications, and proof-only and code-and-proof evaluation modes.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.LG"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.13522","pdf":"https://arxiv.org/pdf/2608.13522","project":null,"code":"https://github.com/sunblaze-ucb/vero","data":null,"hfPaper":"https://huggingface.co/papers/2608.13522"},"evidence":{"snippet":"To bridge this gap, we introduce Vero, the first benchmark to evaluate joint implementation and proof synthesis at the repository level.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":50,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13522"},"ranking":{"30d":{"score":64,"rank":41,"coverage":0.85,"confidence":"High"},"90d":{"score":66,"rank":92,"coverage":0.7,"confidence":"Medium"}},"description":"Vero evaluates repository-level verified code generation in Lean 4, with 43 multi-module instances, formal specifications, and proof-only and code-and-proof evaluation modes.","whyItMatters":"It provides a testbed for measuring progress toward repository-scale verified software synthesis, where current agents fall short.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"388e10a5152ca39158cc75b598822a9c50dee75c9bfe67498740462ee46d4da5"},"motivation":"AI agents are increasingly used for programming, but do not provide any guarantee on the correctness of generated code.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13522","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Sunblaze UCB","organizationType":"academic-lab","sourceUrl":"https://github.com/sunblaze-ucb/vero","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_tsugo_41aa5b88","familyId":"bmf_9964a7463fc4","name":"TsuGO","oneLine":"TsuGO is a process-level reasoning benchmark for search efficiency in LLMs using Go life-and-death problems, parsing CoT into search trees and reporting diagnostics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.13221","pdf":"https://arxiv.org/pdf/2608.13221","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.13221"},"evidence":{"snippet":"We introduce TsuGO, a process-level reasoning benchmark for evaluating Search Efficiency in LLM reasoning through Go life-and-death problems.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13221"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TsuGO is a process-level reasoning benchmark for search efficiency in LLMs using Go life-and-death problems, parsing CoT into search trees and reporting diagnostics.","whyItMatters":"It adds search organization as a missing evaluation dimension beyond final-answer accuracy in LLM reasoning benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"65fed3fb15917c25e4c2074bc3b207b246c4207ce6f8d6babebde1b9fd1b6780"},"motivation":"The evaluation of LLM reasoning is moving from final-answer accuracy to process-level assessment, yet existing methods still fail to capture how models plan reasoning paths and allocate reasoning resources--that is, how they organize search.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13221","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_trapsbench_2047e708","familyId":"bmf_bffec585a3f1","name":"TRAPSBench","oneLine":"TRAPSBench is a procedurally generated video benchmark with 1,404 physics pairs to test epistemic restraint in VLMs, using Penalized Epistemic Calibration Score (PECS).","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.13167","pdf":"https://arxiv.org/pdf/2608.13167","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.13167"},"evidence":{"snippet":"We introduce TRAPSBench, a procedurally generated video benchmark of 1,404 matched physics pairs in which a single targeted change renders the outcome undeterminable from the visual evidence.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13167"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TRAPSBench is a procedurally generated video benchmark with 1,404 physics pairs to test epistemic restraint in VLMs, using Penalized Epistemic Calibration Score (PECS).","whyItMatters":"It highlights that VLMs can internally detect when abstention is required but fail to express it, offering a metric for calibration and restraint evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"308917b653506a73ef52456ad12533ea709e909f580053828fc58540113ff1d5"},"motivation":"When visual evidence is occluded or chaotic, models should abstain.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13167","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_scifigbench_9f0ee596","familyId":"bmf_aefd11b3865b","name":"SciFigBench","oneLine":"SciFigBench is a diagnostic VLM benchmark for scientific figure understanding, covering perception, reasoning, and behavioral reliability under uncertainty.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.13267","pdf":"https://arxiv.org/pdf/2608.13267","project":"https://scifigbench.nlp4sci.com","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.13267"},"evidence":{"snippet":"We introduce SciFigBench, a diagnostic VLM benchmark for scientific figure understanding that jointly evaluates perception, reasoning, and behavioral reliability under uncertainty.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13267"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SciFigBench is a diagnostic VLM benchmark for scientific figure understanding, covering perception, reasoning, and behavioral reliability under uncertainty.","whyItMatters":"It addresses the gap in evaluating VLMs' behavior when visual evidence is missing or misleading, critical for scientific workflows.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ac9e5125c28750f209758db7177029cd0e9805d066722bcf9f5cec57385dc57a"},"motivation":"Existing vision-language model (VLM) benchmarks emphasize perception and reasoning accuracy (how well VLMs describe and reason about what they see in an image), with limited attention to behavioral reliability under uncertainty (how they behave when visual evidence is missing or misleading).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13267","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_polycomp_d2741a8f","familyId":"bmf_0fe1b33f5028","name":"PolyComp","oneLine":"PolyComp is a benchmark for compositional 3D spatial reasoning using polycube problems with 120 problems across four geometry families.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Geometric reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.14741","pdf":"https://arxiv.org/pdf/2608.14741","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.14741"},"evidence":{"snippet":"We introduce PolyComp, a procedurally generated and verified benchmark that stresses visual recognition and compositional spatial reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14741"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PolyComp is a benchmark for compositional 3D spatial reasoning using polycube problems with 120 problems across four geometry families.","whyItMatters":"It stresses visual recognition and compositional reasoning, with random guessing baseline at 25% for challenging multimodal evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4d0aea6734f85d47161b7b78f6452fb39a3d4dd39878bbd63e5491724c296b1f"},"motivation":"We introduce PolyComp, a procedurally generated and verified benchmark that stresses visual recognition and compositional spatial reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14741","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_playworld_54fed7c4","familyId":"bmf_dc419bdc19b6","name":"PlayWorld","oneLine":"PlayWorld evaluates interactive video world models across 171 scenarios with long-horizon objectives. Agent players adaptively control each model, and performance is scored on geometry consistency, interaction fidelity, out-of-sight evolution, and insight evolution, plus basic video quality and controllability metrics.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.13552","pdf":"https://arxiv.org/pdf/2608.13552","project":"https://kxding.github.io/project/PlayWorld/","code":"https://github.com/kxding/PlayWorld","data":null,"hfPaper":"https://huggingface.co/papers/2608.13552"},"evidence":{"snippet":"Building on this paradigm, we introduce PlayWorld, a benchmark providing 171 scenarios, each with a specified objective.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":45,"hfDailySubmittedAt":"2026-08-14T00:00:00.000Z","githubStars":85,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13552"},"ranking":{"30d":{"score":89,"rank":4,"coverage":0.85,"confidence":"High"},"90d":{"score":80,"rank":24,"coverage":0.7,"confidence":"Medium"}},"description":"PlayWorld evaluates interactive video world models across 171 scenarios with long-horizon objectives. Agent players adaptively control each model, and performance is scored on geometry consistency, interaction fidelity, out-of-sight evolution, and insight evolution, plus basic video quality and controllability metrics.","whyItMatters":"Standard fixed-action evaluations fail to compare world models that handle actions differently. PlayWorld provides an objective-driven protocol that reflects real user interaction, enabling fair cross-model assessment and highlighting limitations in spatial consistency and state evolution for long-horizon tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8ec6430d7c17387d79353c7e4f6c3b55dd28417365f2b8dd3d5533c88d917fe9"},"motivation":"Video world models simulate future states conditioned on current observations and user actions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13552","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_naru_dcb9acb4","familyId":"bmf_62e65dd5c196","name":"NARU","oneLine":"NARU evaluates multimodal models on Japanese long-form video understanding across narrative evolution (character evolution, sequential flow, plot progression, thematic development) and cultural nuance (aizuchi, reading the air, subtext, cultural context, sentiment) via multiple-choice QA.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.13210","pdf":"https://arxiv.org/pdf/2608.13210","project":"https://ma-labo.github.io/naru/","code":"https://github.com/infinimind-inc/naru_benchmark","data":null,"hfPaper":"https://huggingface.co/papers/2608.13210"},"evidence":{"snippet":"To address this gap, we introduce NARU, a benchmark designed to evaluate Narrative evolution and Reasoning on cultural Understanding in Japanese long-form video.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":"2026-08-21T00:00:00.000Z","githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13210"},"ranking":{"30d":{"score":62,"rank":42,"coverage":0.85,"confidence":"High"},"90d":{"score":49,"rank":196,"coverage":0.7,"confidence":"Medium"}},"description":"NARU evaluates multimodal models on Japanese long-form video understanding across narrative evolution (character evolution, sequential flow, plot progression, thematic development) and cultural nuance (aizuchi, reading the air, subtext, cultural context, sentiment) via multiple-choice QA.","whyItMatters":"NARU fills a gap in video QA benchmarks by jointly testing long-range narrative tracking and culturally grounded reasoning, which are absent in existing short-horizon or English-centric benchmarks. It provides a rigorous, reproducible protocol for measuring model capabilities in high-context media, supporting model development and product evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-22T11:34:55.479945Z","inputHash":"62498a267d98c1ff82a21e867814665020b942e0ce2b724263a3fff78ebe39be"},"motivation":"Long-form video understanding encompasses tasks that go beyond retrieving isolated events, including tracking an evolving narrative and interpreting social meaning that may remain implicit.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13210","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Infinimind Inc.","organizationType":"company-research-lab","sourceUrl":"https://github.com/infinimind-inc/naru_benchmark","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_longearth-bench_e5b3d65a","familyId":"bmf_3dd2e8ddc733","name":"LongEarth-Bench","oneLine":"LongEarth-Bench evaluates vision-language models on long-horizon Earth observation reasoning, with ~120k QA samples from 117k images, sequences avg 15.14 frames (up to 30), covering 12 tasks in evolution summarization, spatial reasoning, anomaly identification, and logical prediction.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.13344","pdf":"https://arxiv.org/pdf/2608.13344","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.13344"},"evidence":{"snippet":"We introduce LongEarth-Bench, a benchmark containing approximately 120k question-answering samples derived from 117k unique images.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13344"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LongEarth-Bench evaluates vision-language models on long-horizon Earth observation reasoning, with ~120k QA samples from 117k images, sequences avg 15.14 frames (up to 30), covering 12 tasks in evolution summarization, spatial reasoning, anomaly identification, and logical prediction.","whyItMatters":"LongEarth-Bench addresses the lack of benchmarks for long-sequence Earth observation reasoning, providing a more realistic and challenging evaluation for models designed to analyze multi-stage geographic changes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a093c857c6fcc345c6995a9d82ac45ad0f3f780c327347bdce530f8ea5b81003"},"motivation":"Long-horizon Earth observation reasoning requires models to organize multi-stage geographic evolution, localize spatial changes, detect temporal anomalies, and infer future from extended image sequences.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13344","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ligbench_47189b6a","familyId":"bmf_4c9a86e62e01","name":"LigBench","oneLine":"LigBench is an automated evaluation benchmark for AI research idea generation, using fine-grained and reliable evaluation across generation distributions. It includes PAIR-IQ, a dataset for training pairwise idea judgment models.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.13136","pdf":"https://arxiv.org/pdf/2608.13136","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.13136"},"evidence":{"snippet":"To address this challenge, we propose LigBench, an automated evaluation benchmark that enables fine-grained and reliable evaluation of AI research ideas, consistently applicable across different generation distributions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13136"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LigBench is an automated evaluation benchmark for AI research idea generation, using fine-grained and reliable evaluation across generation distributions. It includes PAIR-IQ, a dataset for training pairwise idea judgment models.","whyItMatters":"Current evaluation of research idea generation is fragmented and lacks objective standards. LigBench aims to provide stable, interpretable, and expert-aligned assessments to support scalable and objective evaluation in this emerging area.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e331c59c98129cd0071b460dfb9bb4845dc59ec2eceaedf25f39863c4730f9bf"},"motivation":"With the rapid advancement of large language models (LLMs), research idea generation has attracted increasing attention.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13136","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_humanoidvln_60fdcf09","familyId":"bmf_622a7fe4192c","name":"HumanoidVLN","oneLine":"Evaluates vision-language navigation for humanoid robots across four embodiments in physics-grounded simulator scenarios. Includes 933 episodes with instructions and multiple stylistic variants, assessing success rate and normalized Dynamic Time Warping.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Multimodal"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12860","pdf":"https://arxiv.org/pdf/2608.12860","project":"https://humanoid-vln.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12860"},"evidence":{"snippet":"We present HumanoidVLN, a physics-grounded simulator and benchmark for VLN across diverse humanoid embodiments.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12860"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates vision-language navigation for humanoid robots across four embodiments in physics-grounded simulator scenarios. Includes 933 episodes with instructions and multiple stylistic variants, assessing success rate and normalized Dynamic Time Warping.","whyItMatters":"Addresses the gap in VLN benchmarks for bipedal locomotion and diverse morphologies, providing a platform to compare navigation models under physical constraints and supporting sim-to-real transfer studies.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"25e0707a2701b98970ab65cf7e8042cee518170d985ba6ae841a31c825289576"},"motivation":"Vision-Language Navigation (VLN) for humanoid robots poses challenges existing benchmarks fail to address: bipedal locomotion imposes physical constraints absent from wheeled agents, humanoid morphologies vary across platforms, and egocentric observations are distorted by locomotion-induced camera dynamics.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12860","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HumanoidVLN team","organizationType":"benchmark-organization","sourceUrl":"https://humanoid-vln.github.io/","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_hounsbench_1139158f","familyId":"bmf_5e09e564fc52","name":"HounsBench","oneLine":"HounsBench is a CT-centric patient-state benchmark evaluating three task families: readout, reconstruction, and simulation. It provides patient-disjoint splits and per-family metrics for evaluating models on volumetric medical images and clinical language.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12904","pdf":"https://arxiv.org/pdf/2608.12904","project":null,"code":"https://github.com/byhwhite/HounsWorld.git","data":null,"hfPaper":"https://huggingface.co/papers/2608.12904"},"evidence":{"snippet":"To operationalize this view, we introduce HounsBench, a computed tomography (CT) centric patient-state benchmark that unifies these three task families with patient-disjoint splits and per-family metrics, and HounsWorld, a 3B multimodal world model that treats volumetric scans and language as observations of the shared state through Joint Understanding-Generation Learning.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12904"},"ranking":{"30d":{"score":59,"rank":46,"coverage":0.55,"confidence":"Low"},"90d":{"score":46,"rank":217,"coverage":0.55,"confidence":"Low"}},"description":"HounsBench is a CT-centric patient-state benchmark evaluating three task families: readout, reconstruction, and simulation. It provides patient-disjoint splits and per-family metrics for evaluating models on volumetric medical images and clinical language.","whyItMatters":"HounsBench addresses the evaluation gap for CT-centered intelligence by unifying diverse tasks under a shared patient-state inference framework, enabling assessment of models that must integrate imaging and language for clinical decision support.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4b246dd78ed2bf04c1d377f54fe27c005aff81346d8028d094ad13239f99da87"},"motivation":"Clinical intelligence requires estimating a patient's underlying condition from incomplete observations rather than learning isolated mappings from scans to answers.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12904","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HounsWorld Project","organizationType":"academic-lab","sourceUrl":"https://github.com/byhwhite/HounsWorld.git","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_h2r-bench_da8120f6","familyId":"bmf_6a015e569fc5","name":"H2R-Bench","oneLine":"H2R-Bench evaluates cross-embodiment human-to-robot manipulation video generation. Models convert egocentric human demonstrations into robot manipulation videos under specified target embodiments. Scoring covers five dimensions: goal-state completion, action-event completion, functional contact transfer, embodiment correctness, and general video quality, aggregated into H2RCore.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.13049","pdf":"https://arxiv.org/pdf/2608.13049","project":null,"code":"https://github.com/Rongdingyi/H2R-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.13049"},"evidence":{"snippet":"Therefore, we introduce H2R-Bench, a benchmark for evaluating cross-embodiment human-to-robot manipulation video generation, where models transform egocentric human demonstrations into robot manipulation videos under specified embodiments.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":19,"hfDailySubmittedAt":"2026-08-14T00:00:00.000Z","githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13049"},"ranking":{"30d":{"score":65,"rank":38,"coverage":0.85,"confidence":"High"},"90d":{"score":48,"rank":210,"coverage":0.7,"confidence":"Medium"}},"description":"H2R-Bench evaluates cross-embodiment human-to-robot manipulation video generation. Models convert egocentric human demonstrations into robot manipulation videos under specified target embodiments. Scoring covers five dimensions: goal-state completion, action-event completion, functional contact transfer, embodiment correctness, and general video quality, aggregated into H2RCore.","whyItMatters":"Assesses whether video world models can bridge the embodiment gap between human hands and robotic end-effectors, providing a diagnostic for how well generated videos transfer functional interactions and task execution. Results show generic video quality does not correlate with transfer validity, aiding model selection for robot learning from human video.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ff0b215ab2661583ad04130b940ce1173170b802082fd527b01890a9a7ab5e0a"},"motivation":"Large-scale manipulation data is essential for robot learning, yet collecting robot demonstrations remains expensive and difficult to scale.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13049","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Shanghai Jiao Tong University","organizationType":"academic-lab","sourceUrl":"https://github.com/Rongdingyi/H2R-Bench","role":"benchmark-publisher"},{"name":"Shanghai Artificial Intelligence Laboratory","organizationType":"academic-lab","sourceUrl":"https://github.com/Rongdingyi/H2R-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_egomonth_17f19a53","familyId":"bmf_dfcc09f0ca92","name":"EgoMonth","oneLine":"EgoMonth benchmarks month-level egocentric video understanding with 300+ hours from 20 participants over 20-120 days, 1,443 QA pairs, and a 14-task framework across schema consolidation, episodic indexing, and cascading reasoning.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.13113","pdf":"https://arxiv.org/pdf/2608.13113","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.13113"},"evidence":{"snippet":"We introduce EgoMonth, the first month-level egocentric video understanding benchmark.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13113"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"EgoMonth benchmarks month-level egocentric video understanding with 300+ hours from 20 participants over 20-120 days, 1,443 QA pairs, and a 14-task framework across schema consolidation, episodic indexing, and cascading reasoning.","whyItMatters":"Existing long-video benchmarks lack inter-clip spatiotemporal continuity, so they cannot assess memory across days or weeks. EgoMonth provides a temporal-grounded evaluation for long-term memory in MLLMs, revealing that even top models remain far below human performance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c972f9445040cd30ad8721a8f6f2cf010362aaef364b8d13be0f7c5b0677817e"},"motivation":"Recent advances in Multimodal Large Language Models (MLLMs) have led to substantial progress in video understanding, accompanied by a growing number of long video benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13113","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_edit2tikz_4730ba41","familyId":"bmf_95a10c6e66b2","name":"Edit2TikZ","oneLine":"Edit2TikZ evaluates instruction-guided scientific figure editing with TikZ code, featuring 1,548 samples with textual or visual localization requests and multi-step edits, using a human-aligned evaluation framework to measure edit completion and content preservation.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.13441","pdf":"https://arxiv.org/pdf/2608.13441","project":null,"code":"https://github.com/Solunny/Edit2TikZ","data":null,"hfPaper":"https://huggingface.co/papers/2608.13441"},"evidence":{"snippet":"We introduce Edit2TikZ, a comprehensive benchmark for scientific figure editing tasks, featuring 1,548 diverse and high-quality samples.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.13441"},"ranking":{"30d":{"score":34,"rank":129,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":352,"coverage":0.55,"confidence":"Low"}},"description":"Edit2TikZ evaluates instruction-guided scientific figure editing with TikZ code, featuring 1,548 samples with textual or visual localization requests and multi-step edits, using a human-aligned evaluation framework to measure edit completion and content preservation.","whyItMatters":"Existing benchmarks focus on figure reconstruction or generation, leaving a gap for systematic evaluation of instruction-guided editing with compilable code. This benchmark provides a standardized protocol for assessing models' ability to perform precise, code-based edits while preserving unrelated content, offering practical value for developing reliable multimodal systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c72b462be79393ac1f33e98bd8b2621eb4ba990f93da0280006cf195e2551067"},"motivation":"Although multimodal large language models (MLLMs) have shown substantial potential in visual understanding and graphic code generation, editing scientific figures through code presents a greater challenge: a model must jointly recover visual structure, ground the requested change, generate compilable code, and preserve all unrelated content.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.13441","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Solunny","organizationType":"community","sourceUrl":"https://github.com/Solunny/Edit2TikZ","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_docprivacybench_1e095c41","familyId":"bmf_db22e09b280e","name":"DocPrivacyBench","oneLine":"DocPrivacyBench evaluates susceptibility of document understanding MLLMs to relational privacy leakage when visual evidence is absent or minimal, using KIE tasks on identity documents.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12911","pdf":"https://arxiv.org/pdf/2608.12911","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12911"},"evidence":{"snippet":"It suppresses the leakage of high-risk field pairs while preserving KIE performance.Second, we introduce DocPrivacyBench, a novel benchmark to systematically evaluate a model's susceptibility to privacy leakage under conditions of absent or minimal visual evidence.Third, we evaluate three MLLMs and six unlearning methods using this benchmark, assessing both post-unlearning leakage suppression and utility preservation.Our results demonstrate that existing MLLMs consistently exhibit privacy leakag","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12911"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"DocPrivacyBench evaluates susceptibility of document understanding MLLMs to relational privacy leakage when visual evidence is absent or minimal, using KIE tasks on identity documents.","whyItMatters":"Addresses underexplored privacy vulnerabilities in document MLLMs, providing a framework to assess and mitigate leakage of correlated sensitive fields, which is crucial for trustworthy deployment in document processing.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b077a7d75b0671f82623b40664d07463f3e8457ba5233ff336ba1821edca71df"},"motivation":"While the privacy risks of multimodal large language models (MLLMs) have drawn significant attention, the unique vulnerabilities of domain-specific MLLMs remain largely underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12911","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_comedbench_50407db9","familyId":"bmf_572ffe3d9a69","name":"CoMedBench","oneLine":"CoMedBench evaluates synthetic medical data fidelity and downstream utility across 37 dataset-task pairs from seven public sources, using a common clinical-validity framework and shared training and evaluation engine.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.LG"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12805","pdf":"https://arxiv.org/pdf/2608.12805","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12805"},"evidence":{"snippet":"We introduce CoMedBench, a reproducible benchmark that evaluates a family of generators under a common clinical-validity framework and one shared training and evaluation engine, spanning static tabular and temporal downstream tasks on established critical-care datasets.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12805"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CoMedBench evaluates synthetic medical data fidelity and downstream utility across 37 dataset-task pairs from seven public sources, using a common clinical-validity framework and shared training and evaluation engine.","whyItMatters":"Provides a reproducible benchmark for comparing synthetic data generators across multiple datasets and tasks, addressing the lack of comprehensive evaluation in prior studies and aiding in deciding when synthetic data is viable for model development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3dee9274de668d7383680dcaf1e8bc0c99f5dcd98d549e0d8e80998c575d1493"},"motivation":"Access to clinical data is essential for developing reliable healthcare machine learning systems, but direct use of electronic health records is constrained by privacy regulation, institutional review, data-use agreements, and the risk of re-identification.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12805","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_bavground_960d10f0","familyId":"bmf_dedb900e4843","name":"BavGround","oneLine":"BavGround evaluates regional cultural grounding and dialect competence in Bavarian across English, German, and Bavarian, with 618 multi-parallel multiple-choice questions across eight cultural domains.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12894","pdf":"https://arxiv.org/pdf/2608.12894","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12894"},"evidence":{"snippet":"We introduce BavGround, a benchmark for evaluating Bavarian regional cultural grounding and dialect competence across English, German and Bavarian.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12894"},"ranking":{"30d":{"score":47,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"BavGround evaluates regional cultural grounding and dialect competence in Bavarian across English, German, and Bavarian, with 618 multi-parallel multiple-choice questions across eight cultural domains.","whyItMatters":"Addresses the gap in cultural evaluation for regional and dialect communities, providing a protocol-aware benchmark that highlights performance differences across evaluation methods and supports localized assessment of LLMs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"170f8b763b731772f208a0399c8a4fb75882b25af7d60e0e4633e969df0e9322"},"motivation":"Cultural evaluation of large language models (LLMs) often focuses on high-resource standard languages, leaving regional culture and dialect communities underrepresented.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12894","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_videogaia_c89526e7","familyId":"bmf_be79630d2651","name":"VideoGAIA","oneLine":"VideoGAIA evaluates agentic video understanding through multi-turn, tool-augmented interactions where models must iteratively perceive videos, invoke external tools, and integrate multimodal evidence. It contains 271 human-verified tasks across diverse real-world scenarios.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.14718","pdf":"https://arxiv.org/pdf/2608.14718","project":null,"code":"https://github.com/zfkarl/VideoGAIA","data":null,"hfPaper":"https://huggingface.co/papers/2608.14718"},"evidence":{"snippet":"Towards this end, we introduce VideoGAIA, an agentic video understanding benchmark for general artificial intelligence (AI) assistants.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":10,"hfDailySubmittedAt":"2026-08-18T00:00:00.000Z","githubStars":8,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14718"},"ranking":{"30d":{"score":71,"rank":29,"coverage":0.85,"confidence":"High"},"90d":{"score":60,"rank":137,"coverage":0.7,"confidence":"Medium"}},"description":"VideoGAIA evaluates agentic video understanding through multi-turn, tool-augmented interactions where models must iteratively perceive videos, invoke external tools, and integrate multimodal evidence. It contains 271 human-verified tasks across diverse real-world scenarios.","whyItMatters":"Conventional single-turn video understanding benchmarks are becoming saturated; VideoGAIA moves beyond to assess advanced MLLMs' ability to act as general AI assistants, using tools to gather complementary information across turns. It provides a timely, challenging benchmark for next-generation video understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7d365d3edbf1067bad146b71b70188fdbdf07060e1d52b2a5e61a2de2f371938"},"motivation":"Video understanding is a fundamental task for evaluating the capabilities of multimodal large language models (MLLMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14718","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Karl28","organizationType":"community","sourceUrl":"https://github.com/zfkarl/VideoGAIA","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_strad_bb4cb514","familyId":"bmf_48621e764b76","name":"StrAD","oneLine":"StrAD is a benchmark for long-form audio description generation on full-length videos across diverse genres, reformulating the task as streaming dense video captioning without ground-truth timestamps.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12549","pdf":"https://arxiv.org/pdf/2608.12549","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12549"},"evidence":{"snippet":"We introduce StrAD, a benchmark for long-form AD generation on full-length videos spanning diverse genres such as movies, documentaries, short films, performances, and video games.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12549"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"StrAD is a benchmark for long-form audio description generation on full-length videos across diverse genres, reformulating the task as streaming dense video captioning without ground-truth timestamps.","whyItMatters":"Addresses the lack of evaluation for full-video AD generation, providing a measurable benchmark that includes both fine-tuned and zero-shot approaches, which is essential for scaling accessibility.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5f039f2caf09668bca00d8ee9a5aa4e283525258425625bb09bec64edbe4e345"},"motivation":"Visual content is the dominant medium of communication, yet without audio descriptions (ADs), it remains inaccessible to blind and low-vision people.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12549","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_steerbench-work_754206df","familyId":"bmf_9f160b21637e","name":"SteerBench-Work","oneLine":"SteerBench-Work evaluates agent steering decisions at action boundaries in workplace scenarios across seven domains, with 106 incident-anchored scenarios and evidence-reversed mirrors, scored on correct proceed/hold boundaries.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences","Finance & Economics"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech","Financial Services"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12654","pdf":"https://arxiv.org/pdf/2608.12654","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12654"},"evidence":{"snippet":"We introduce SteerBench-Work, an incident-anchored, bidirectional benchmark for that decision in workplace agents across developer operations, customer service, finance, legal, medical, HR, and security.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12654"},"ranking":{"30d":{"score":47,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SteerBench-Work evaluates agent steering decisions at action boundaries in workplace scenarios across seven domains, with 106 incident-anchored scenarios and evidence-reversed mirrors, scored on correct proceed/hold boundaries.","whyItMatters":"Addresses the critical pre-commit decision in long-running agents, where a single step can have significant consequences. The benchmark reveals that models tend to over-refuse authorized actions, providing valuable insight for calibrating agent behavior to avoid both unsafe actions and unnecessary delays.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c5c73171cb06508b48d45dd868634ea611d199ecd1014db18559877225383f61"},"motivation":"Long-running LLM agents act through tools, and a single step can send an email, merge a pull request, or wire a payment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12654","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"cross-domain"},{"id":"bm_s2r-bench_847c5139","familyId":"bmf_912689c7de1c","name":"S2R-Bench","oneLine":"S2R-Bench evaluates video reflection removal, supporting full-reference and human perceptual assessment, and is used to validate the S2R-Removal model.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.11562","pdf":"https://arxiv.org/pdf/2608.11562","project":"https://codingwzp.github.io/VideoDereflection_S2R","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.11562"},"evidence":{"snippet":"We further build S2R-Bench, the first benchmark for video reflection removal, supporting both full-reference evaluation and real-world human perceptual assessment.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":10,"hfDailySubmittedAt":"2026-08-13T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.11562"},"ranking":{"30d":{"score":55,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":54,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"S2R-Bench evaluates video reflection removal, supporting full-reference and human perceptual assessment, and is used to validate the S2R-Removal model.","whyItMatters":"Addresses the lack of benchmarks for video reflection removal, providing a dedicated evaluation protocol that includes both synthetic and real-world scenarios, which is crucial for advancing this under-explored task.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e02a15f244a3753241998f5d1a4998b30b8f4debed4f6ae1f24eb39d95245803"},"motivation":"Videos captured through glass often contain reflections that degrade visual quality and interfere with downstream vision tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.11562","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_realistictritonbench_fb357880","familyId":"bmf_b1807ea10af3","name":"RealisticTritonBench","oneLine":"RealisticTritonBench evaluates LLM-generated Triton kernels using tasks derived from real-world pull requests in popular AI frameworks, with end-to-end integration tests.","area":"Code & Software","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12004","pdf":"https://arxiv.org/pdf/2608.12004","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12004"},"evidence":{"snippet":"To address these limitations, we introduce RealisticTritonBench, the first benchmark to derive Triton kernel generation tasks from real-world pull requests in popular AI frameworks, enabling realistic, production-like evaluation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12004"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RealisticTritonBench evaluates LLM-generated Triton kernels using tasks derived from real-world pull requests in popular AI frameworks, with end-to-end integration tests.","whyItMatters":"Existing benchmarks focus on isolated kernel translation and may have flawed evaluation scripts, while this benchmark provides realistic tasks and robust end-to-end evaluation, offering more practical insight into LLM performance for production kernel development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4245259960ec166ccdb4f1915bdc6d8dc983a5fcc26fa5b5e394d7e5c36877f0"},"motivation":"In modern AI frameworks, GPU kernels are key to overall system performance.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ASE 2026","evidence":"Accepted by ASE 2026","evidenceUrl":"https://arxiv.org/abs/2608.12004","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ASE 2026","reviewStatus":"accepted","decisionRaw":"Accepted by ASE 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.12004","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by ASE 2026","level":"author-claim"}]}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_netlistbench_58d2657e","familyId":"bmf_3c373592114f","name":"NetlistBench","oneLine":"NetlistBench evaluates LLM reliability in recognizing and manipulating SPICE netlists, covering parameter and connectivity recognition, edits, hierarchical operations, equivalence judgment, and compound editing, with deterministic structure-aware scoring.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12197","pdf":"https://arxiv.org/pdf/2608.12197","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12197"},"evidence":{"snippet":"We present \\textbf{NetlistBench}, a structure-verified benchmark for SPICE netlist recognition and manipulation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12197"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"NetlistBench evaluates LLM reliability in recognizing and manipulating SPICE netlists, covering parameter and connectivity recognition, edits, hierarchical operations, equivalence judgment, and compound editing, with deterministic structure-aware scoring.","whyItMatters":"It addresses the gap in assessing LLM reliability for simulator-facing netlist tasks, distinct from high-level design reasoning, and provides a structured evaluation to guide trustworthy circuit design automation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"690dac1af652a91d5cdcc0ce216c9c61fd24dddc9bbcf0cfeadbcdfa0163eadf"},"motivation":"Large Language Models (LLMs) are increasingly used in circuit design workflows, yet their reliability on simulator-facing SPICE netlist recognition and manipulation remains poorly understood and is rarely separated from high-level design reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"MLCAD 2026","evidence":"accepted by MLCAD 2026","evidenceUrl":"https://arxiv.org/abs/2608.12197","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"MLCAD 2026","reviewStatus":"accepted","decisionRaw":"accepted by MLCAD 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.12197","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"accepted by MLCAD 2026","level":"author-claim"}]}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_frontierfinance_f2f44751","familyId":"bmf_d3024ba14951","name":"FrontierFinance","oneLine":"Comprises 220 expert-crafted queries with 11,543 source-attributed rubrics across six finance use cases, evaluating agents on open-ended analyst-style answers via rubric-based grading.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.11683","pdf":"https://arxiv.org/pdf/2608.11683","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.11683"},"evidence":{"snippet":"We introduce FrontierFinance, a fully open benchmark of 220 expert-crafted queries and 11,543 source-attributed rubrics spanning six crucial use cases across the full investor workflow.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.11683"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Comprises 220 expert-crafted queries with 11,543 source-attributed rubrics across six finance use cases, evaluating agents on open-ended analyst-style answers via rubric-based grading.","whyItMatters":"Captures the full investor workflow beyond narrow data extraction, offering a harder and broader evaluation that assesses agent quality and efficiency in realistic finance research.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"7fd3ddaacac95ccc812e5910cf0938dc4d139d8776cfa1f2e38ca412fda17376"},"motivation":"AI agents are increasingly deployed for professional investment research, yet no benchmark captures the complexity of the full investor workflow.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.11683","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Samaya AI","organizationType":"company-research-lab","sourceUrl":"https://arxiv.org/abs/2608.11683","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_enterpriserag_49e3b609","familyId":"bmf_11382aad9957","name":"EnterpriseRAG","oneLine":"The benchmark evaluates LLM instruction adherence and robustness in enterprise retrieval scenarios, using 983 expert-validated samples across six domains, simulating retrieval noise, knowledge gaps, and factual conflicts.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval","Robustness","Factuality"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.11584","pdf":"https://arxiv.org/pdf/2608.11584","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.11584"},"evidence":{"snippet":"We introduce EnterpriseRAG, a benchmark of 983 expert-validated samples across six domains that systematically simulates three failure modes absent from prior work: retrieval noise, knowledge gaps, and factual conflicts, coupled with complex instructions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.11584"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"The benchmark evaluates LLM instruction adherence and robustness in enterprise retrieval scenarios, using 983 expert-validated samples across six domains, simulating retrieval noise, knowledge gaps, and factual conflicts.","whyItMatters":"Existing RAG benchmarks assume clean retrieval and simple queries, failing to capture production conditions. This benchmark addresses the gap by measuring holistic compliance under non-ideal conditions, informing deployment decisions for enterprise-scale RAG systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0d95abb771fcce5a6b9fac9c7f67ebbd1c0e79f449ded04813552f41eefac32a"},"motivation":"Enterprise RAG deployments face a critical reliability gap: while LLMs satisfy 80% of individual constraints, only 26.8% of responses meet all requirements simultaneously, revealing a 57-point orchestration gap.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.11584","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness","Search & Retrieval"],"domainScope":"general"},{"id":"bm_dig-bench_37529e7b","familyId":"bmf_78f64581e492","name":"DiG-bench","oneLine":"Evaluates AI agents on discovery of unknown transformation rules across 70 independent games with seven difficulty tiers, requiring experimentation and rule generalization.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.12593","pdf":"https://arxiv.org/pdf/2608.12593","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12593"},"evidence":{"snippet":"To address this gap, we release a new benchmark: DiG-bench (Discovery in Games).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12593"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates AI agents on discovery of unknown transformation rules across 70 independent games with seven difficulty tiers, requiring experimentation and rule generalization.","whyItMatters":"Fills the gap in benchmarks for scientific discovery in controlled environments, testing the capacity to formulate novel generalizations through interaction.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"8b088888b08ff4e9e027e58c7b770b09d818da4c50254aac13ffcf3e287c7c8f"},"motivation":"Discovery---formulating novel generalizations---is a central part of the scientific process.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12593","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DiG-bench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.12593","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_diagram-mmu_a41bffa4","familyId":"bmf_39ca0919aed5","name":"Diagram-MMU","oneLine":"Diagram-MMU evaluates multimodal language models on parsing scientific diagrams into LaTeX TikZ code, editing diagram code, and answering questions about diagrams, using 3.7k diagrams and 18.3k questions across six domains.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12262","pdf":"https://arxiv.org/pdf/2608.12262","project":"https://vi-ocean.github.io/projects/diagram-mmu","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12262"},"evidence":{"snippet":"In this paper, we build a benchmark, Diagram-MMU, a multi-modal benchmark designed to assess MLLMs' ability for scientific diagram parsing and understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12262"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Diagram-MMU evaluates multimodal language models on parsing scientific diagrams into LaTeX TikZ code, editing diagram code, and answering questions about diagrams, using 3.7k diagrams and 18.3k questions across six domains.","whyItMatters":"The evaluation provides insight into model capabilities on diagram-to-code generation tasks, a practical need in scientific authoring tools, and identifies gaps between reasoning and code generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1adae049c186596aba9d2f70186ac67ca1a4334d31facfbf4a63cddbd6526ca2"},"motivation":"Multimodal Large Language Models (MLLMs) have been growing the capability for scientific writing and collaboration.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12262","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ctbench_d210f8ea","familyId":"bmf_a5ad32879a12","name":"CTBench","oneLine":"Evaluates AI agents on telecom network troubleshooting tasks, focusing on root cause analysis and path restoration, using expert-grounded metrics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12002","pdf":"https://arxiv.org/pdf/2608.12002","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12002"},"evidence":{"snippet":"In this paper, we introduce CTBench, a public benchmark for assessing whether an agent behaves like a competent telecom troubleshooting engineer.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12002"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates AI agents on telecom network troubleshooting tasks, focusing on root cause analysis and path restoration, using expert-grounded metrics.","whyItMatters":"Fills the gap in evaluating AI agents for realistic telecom operations, emphasizing evidence-based diagnosis and practical constraints.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fad9dec77a807219b275ceb8cd3b26136f319213f2e66a328a5f67f484c238a7"},"motivation":"Agents are increasingly considered for automating network operations and maintenance, where engineers must diagnose network faults, optimize configurations to enhance services, and reduce operational costs while acting under strict constraints.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12002","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_aeroground_3ddedf6a","familyId":"bmf_0dd007a3643a","name":"AeroGround","oneLine":"AeroGround evaluates vision-language models on aerial-ground collaborative reasoning using a simulated dataset of ~29,000 multimodal observation groups and 2,250 QA instances covering cross-view correspondence, spatial understanding, and reasoning.","area":"Vision & 3D","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.14721","pdf":"https://arxiv.org/pdf/2608.14721","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.14721"},"evidence":{"snippet":"To address this gap, we introduce AeroGround, a comprehensive benchmark for evaluating VLMs in aerial-ground collaborative reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14721"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AeroGround evaluates vision-language models on aerial-ground collaborative reasoning using a simulated dataset of ~29,000 multimodal observation groups and 2,250 QA instances covering cross-view correspondence, spatial understanding, and reasoning.","whyItMatters":"Existing UAV benchmarks focus on aerial-only views; AeroGround fills the gap for aerial-ground collaboration, offering a standardized evaluation for models in real-world applications like rescue and inspection, with clear human performance comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ebe3a22902cb80d2177531bb1b4faed54b18b91302869e46a93fd79adfcdedfe"},"motivation":"Vision-language models (VLMs) have been widely employed in understanding and reasoning tasks for unmanned aerial vehicles (UAVs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14721","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_voxsumm_b7118a8e","familyId":"bmf_d932f4e6d176","name":"VoxSumm","oneLine":"VoxSumm evaluates joint summarization and translation of long-form spoken news. It comprises 10,045 BBC article-summary pairs across 24 languages and about 703 hours of speech, with scoring based on summarization and translation quality.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SD"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.10359","pdf":"https://arxiv.org/pdf/2608.10359","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.10359"},"evidence":{"snippet":"We additionally introduce VoxSumm, the first multilingual and cross-lingual benchmark for this task, comprising 10,045 BBC article-summary pairs across 24 languages and encompassing approximately 703 hours of speech data.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.10359"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"VoxSumm evaluates joint summarization and translation of long-form spoken news. It comprises 10,045 BBC article-summary pairs across 24 languages and about 703 hours of speech, with scoring based on summarization and translation quality.","whyItMatters":"Existing benchmarks treat long-document summarization and speech translation separately, leaving a gap for cross-lingual summarization of spoken content. VoxSumm provides a publicly inspectable resource for developing systems that compress and translate long-form speech, aiding evaluation of multilingual instruction-following and cross-lingual generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"902cb002d670db7e66be50d926cf2e9e67139ff4e4bc8d07dece8efd01e1f473"},"motivation":"As information increasingly traverses linguistic boundaries, users require concise cross-lingual representations of long-form content.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.10359","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"VoxSumm Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.10359","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_viseditbench_e7b0411b","familyId":"bmf_e2e3088b5cf4","name":"VisEditBench","oneLine":"VisEditBench is a benchmark comprising 1,395 human-annotated visualization code-editing tasks across two settings: feedback-guided repair and reference-guided restyling. Models are evaluated on their ability to revise existing visualization code based on multimodal feedback such as buggy or marked charts with textual instructions, and target chart images. Scoring is based on pass rates of generated code executions.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.10408","pdf":"https://arxiv.org/pdf/2608.10408","project":null,"code":"https://github.com/vis-nlp/VisEditBench","data":null,"hfPaper":"https://huggingface.co/papers/2608.10408"},"evidence":{"snippet":"We introduce VisEditBench, a benchmark of 1,395 human-annotated visualization code-editing tasks grounded in realistic visualization workflows and failure cases.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.10408"},"ranking":{"30d":{"score":56,"rank":51,"coverage":0.55,"confidence":"Low"},"90d":{"score":44,"rank":241,"coverage":0.55,"confidence":"Low"}},"description":"VisEditBench is a benchmark comprising 1,395 human-annotated visualization code-editing tasks across two settings: feedback-guided repair and reference-guided restyling. Models are evaluated on their ability to revise existing visualization code based on multimodal feedback such as buggy or marked charts with textual instructions, and target chart images. Scoring is based on pass rates of generated code executions.","whyItMatters":"Existing benchmarks focus on generating visualizations from scratch, leaving the iterative editing process unexplored. VisEditBench provides a standardized evaluation for this practical task, enabling comparisons across VLMs and highlighting gaps in open-source models, particularly in visually grounded style adaptation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0140f504284b616dcae006538da4045f8c23ddf98da04ed0a9a58224967c4c7e"},"motivation":"Vision-language models (VLMs) have shown strong capabilities in generating visualization code from textual or visual specifications.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.10408","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"vis-nlp","organizationType":"academic-lab","sourceUrl":"https://github.com/vis-nlp/VisEditBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_vibelifebench_bf7626dc","familyId":"bmf_978e50cecb82","name":"VibeLifeBench","oneLine":"VibeLifeBench evaluates LLM agents on 200 long-horizon, multi-week tasks across 10 everyday-life domains in a simulated world of 22 mock services. Agents must manage silent world changes and implicit constraints, with fine-grained weighted checks on end state, timeliness, and constraint adherence.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.10875","pdf":"https://arxiv.org/pdf/2608.10875","project":null,"code":"https://github.com/evolvent-ai/VibeLifeBench","data":null,"hfPaper":"https://huggingface.co/papers/2608.10875"},"evidence":{"snippet":"We introduce VibeLifeBench, a benchmark of 200 long-horizon tasks across ten everyday-life domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":17,"hfDailySubmittedAt":"2026-08-12T00:00:00.000Z","githubStars":13,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.10875"},"ranking":{"30d":{"score":77,"rank":16,"coverage":0.85,"confidence":"High"},"90d":{"score":65,"rank":101,"coverage":0.7,"confidence":"Medium"}},"description":"VibeLifeBench evaluates LLM agents on 200 long-horizon, multi-week tasks across 10 everyday-life domains in a simulated world of 22 mock services. Agents must manage silent world changes and implicit constraints, with fine-grained weighted checks on end state, timeliness, and constraint adherence.","whyItMatters":"Existing benchmarks focus on short, static tasks, leaving a gap in measuring long-horizon proactive assistance. VibeLifeBench provides a credible public evaluation for agents that must operate over weeks with changing environments, offering practical decision value for deploying personal assistants.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c993e63c4cfcbf56c87edd9bccb6ac81aa608560b84648e6773efb99ddfea1bd"},"motivation":"Large language model (LLM) agents are increasingly deployed as personal assistants.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.10875","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Evolvent AI","organizationType":"company-research-lab","sourceUrl":"https://github.com/evolvent-ai/VibeLifeBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_vialectbench_c8a2ee22","familyId":"bmf_9a6b7205c2c0","name":"VialectBench","oneLine":"Evaluates LLM robustness to Vietnamese dialectal rewrites across emotion recognition, natural language inference, question answering, and multiple-choice QA, measuring performance degradation across six dialect groups.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.10414","pdf":"https://arxiv.org/pdf/2608.10414","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.10414"},"evidence":{"snippet":"We introduce VialectBench (Vietnamese Dialects Benchmarking), a controlled benchmark for testing whether model decisions remain stable across six Vietnamese dialect groups.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.10414"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates LLM robustness to Vietnamese dialectal rewrites across emotion recognition, natural language inference, question answering, and multiple-choice QA, measuring performance degradation across six dialect groups.","whyItMatters":"Highlights that model performance on standard Vietnamese does not guarantee reliable behavior under regional variation, informing deployment decisions for Vietnamese-language applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"170093ed0814dd2d3c3463a8f2410637b47634d5d52ece9765daf3e6872738a5"},"motivation":"Large Language Models (LLMs) are typically evaluated on standard written Vietnamese, yet everyday communication frequently involves regional dialects that preserve meaning but differ in surface form.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.10414","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_tangpoetrybench_c91f571c","familyId":"bmf_f51483afb01a","name":"TangPoetryBench","oneLine":"TangPoetryBench evaluates text-to-image models on illustrating classical Chinese Tang poems across ten human-annotated dimensions, with a rubric-conditioned evaluator (PAE).","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.11452","pdf":"https://arxiv.org/pdf/2608.11452","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.11452"},"evidence":{"snippet":"We introduce TangPoetryBench, a multi-dimensional benchmark of 1,280 images (320 classical Chinese Tang poems x 4 state-of-the-art T2I models) with quality-controlled human annotations across ten dimensions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.11452"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TangPoetryBench evaluates text-to-image models on illustrating classical Chinese Tang poems across ten human-annotated dimensions, with a rubric-conditioned evaluator (PAE).","whyItMatters":"Existing metrics fail to capture cultural and emotional fidelity in poetry-to-image generation; this benchmark provides a multi-dimensional human-annotated dataset and an automated evaluator to support model comparison in this niche domain.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cc83960980390597ab354e5ef5763aa8cfbb047a2d37467427ec2c1d82000353"},"motivation":"Text-to-image (T2I) models are increasingly asked to illustrate literary and cultural content, yet we cannot measure how well an image renders the meaning of a poem.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.11452","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_sre-bench_d394a59b","familyId":"bmf_d2aa6d139e85","name":"SRE-Bench","oneLine":"SRE-Bench evaluates AI agents on reverse engineering of binaries compiled from 19 private real-world-scale C programs (average 16.9K LoC) with 44 anti-analysis primitives, yielding 262 binary instances and 1572 deterministically graded tasks.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.11469","pdf":"https://arxiv.org/pdf/2608.11469","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.11469"},"evidence":{"snippet":"To this end, we introduce SRE-Bench, the first realistic, contamination-free RE benchmark.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.11469"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SRE-Bench evaluates AI agents on reverse engineering of binaries compiled from 19 private real-world-scale C programs (average 16.9K LoC) with 44 anti-analysis primitives, yielding 262 binary instances and 1572 deterministically graded tasks.","whyItMatters":"Existing benchmarks for agentic cybersecurity miss either contamination control or realistic scale. SRE-Bench addresses this gap by combining private, real-world-scale binaries with deterministic grading, enabling reliable measurement of agent performance in binary analysis and highlighting the gap between source-code and binary security capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2aeed24a8279cc24f3368bd4c0dda9ef029042a24563a915d52b0627e3d81ba2"},"motivation":"AI agents are rapidly improving in cybersecurity capabilities when the source code is available for analysis, yet much of the software most consequential to cybersecurity, including malware, firmware, and proprietary applications, is available only as binaries.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.11469","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_spieval_a21bc309","familyId":"bmf_8905e75a39cb","name":"SPIEval","oneLine":"SPIEval is a human-curated benchmark for evaluating large language models as mobile assistants that retrieve and reason over personal information scattered across multiple apps. It comprises 250 tasks across five cognitive capabilities, 4,335 fictional personal records in 10 simulated apps, and supports multi-turn interaction through 21 tools. Models receive underspecified user instructions and must search records and invoke tools; final execution calls are compared to human-annotated gold calls at the parameter level.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.10692","pdf":"https://arxiv.org/pdf/2608.10692","project":null,"code":null,"data":"https://huggingface.co/datasets/Junjie-Ye/SPIEval","hfPaper":"https://huggingface.co/papers/2608.10692"},"evidence":{"snippet":"To address this gap, we introduce SPIEval, a human-curated benchmark grounded in five cognitive capabilities (i.e., reasoning, disambiguation, integration, preference inference, and multi-intent decomposition).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":12,"hfDailySubmittedAt":"2026-08-12T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":226,"hfDatasetLikes":1},"source":{"type":"arxiv","id":"2608.10692"},"ranking":{"30d":{"score":58,"rank":48,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":8,"datasetRankPopulation":20},"90d":{"score":55,"rank":165,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":24,"datasetRankPopulation":51}},"description":"SPIEval is a human-curated benchmark for evaluating large language models as mobile assistants that retrieve and reason over personal information scattered across multiple apps. It comprises 250 tasks across five cognitive capabilities, 4,335 fictional personal records in 10 simulated apps, and supports multi-turn interaction through 21 tools. Models receive underspecified user instructions and must search records and invoke tools; final execution calls are compared to human-annotated gold calls at the parameter level.","whyItMatters":"Existing benchmarks do not target the challenge of leveraging scattered personal information across apps in mobile assistant settings. SPIEval provides a controlled, verifiable evaluation environment grounded in five cognitive capabilities, enabling assessment of model capabilities in realistic mobile contexts. The reported results show substantial room for improvement and highlight fundamental limitations in information localization and search efficiency, offering practical guidance for deploying LLM-based assistants.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"edf6c3676d14db9361b58c1f98994c02d946f5484df4db8cf02829bc5ba991c2"},"motivation":"Large language models (LLMs) are increasingly deployed as mobile assistants, where a key challenge is leveraging personal information scattered across multiple applications (apps) to complete user instructions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.10692","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_prmu_0126a6f3","familyId":"bmf_2cbd078f4d99","name":"PRMU","oneLine":"PRMU evaluates corpus-free multimodal unlearning of person-related knowledge in MLLMs, using textual and visual probes including adversarial evaluation and locality analysis.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.11149","pdf":"https://arxiv.org/pdf/2608.11149","project":null,"code":"https://github.com/2231122/PRMU","data":null,"hfPaper":"https://huggingface.co/papers/2608.11149"},"evidence":{"snippet":"To address this limitation, we introduce PRMU, a benchmark for evaluating corpus-free multimodal unlearning under realistic person-centric deletion requests.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.11149"},"ranking":{"30d":{"score":34,"rank":130,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":353,"coverage":0.55,"confidence":"Low"}},"description":"PRMU evaluates corpus-free multimodal unlearning of person-related knowledge in MLLMs, using textual and visual probes including adversarial evaluation and locality analysis.","whyItMatters":"Addresses realistic deletion scenarios where original corpora are unavailable, providing a way to measure forgetting-locality trade-offs and vulnerability to knowledge reactivation, which is valuable for safe deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"99af0e1c783362a2968058a7c09d159952bec25f2c29ab41333c6380c8f8b792"},"motivation":"Multimodal large language models (MLLMs) have demonstrated remarkable capabilities in storing and recalling rich person-related knowledge, raising increasing concerns about reliable knowledge removal.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.11149","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mpar-bench_22a7bcc7","familyId":"bmf_346c45b511f3","name":"MPAR-Bench","oneLine":"MPAR-Bench evaluates multi-point associative reasoning in LLMs across English and Chinese using 1,000 items with diverse clues. Scoring includes exact-match accuracy, ANLS, embedding similarity, and reasoning-trace verification, with four perturbation types.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.10444","pdf":"https://arxiv.org/pdf/2608.10444","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.10444"},"evidence":{"snippet":"We introduce MPAR-Bench, a bilingual English-Chinese benchmark that isolates reasoning breadth through multi-point associative reasoning.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.10444"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MPAR-Bench evaluates multi-point associative reasoning in LLMs across English and Chinese using 1,000 items with diverse clues. Scoring includes exact-match accuracy, ANLS, embedding similarity, and reasoning-trace verification, with four perturbation types.","whyItMatters":"Current benchmarks focus on reasoning depth (longer chains) but neglect breadth (parallel semantic exploration). MPAR-Bench fills that gap, showing that depth does not guarantee robust breadth, and offers a practical tool for assessing models on a complementary reasoning dimension.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2a8fdf05a2930e2a1a4ae4ff5f395ca2069bd53026909e9c4adc60208cdfc2e2"},"motivation":"Large language models (LLMs) have made substantial progress on reasoning tasks that require increasingly long and complex inferential chains.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.10444","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"No official publisher identified","organizationType":"community","sourceUrl":"https://arxiv.org/abs/2608.10444","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_mobilejudgebench_5adbb014","familyId":"bmf_482470db024a","name":"MobileJudgeBench","oneLine":"MobileJudgeBench evaluates LLM-as-judge methods on mobile agent trajectories. It includes 931 human-annotated trajectories from 6 mobile agent benchmarks, covering 4 agent models and 68 apps.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.11434","pdf":"https://arxiv.org/pdf/2608.11434","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.11434"},"evidence":{"snippet":"We introduce MobileJudgeBench, a benchmark for systematically evaluating LLM-as-judge methods on mobile agent trajectories.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.11434"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MobileJudgeBench evaluates LLM-as-judge methods on mobile agent trajectories. It includes 931 human-annotated trajectories from 6 mobile agent benchmarks, covering 4 agent models and 68 apps.","whyItMatters":"Mobile agent benchmarks increasingly rely on LLM-based judges, yet their reliability is unexamined. MobileJudgeBench fills this gap by providing a standardized evaluation to select reliable judges, improving evaluation fidelity and downstream reinforcement learning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5d44e8805339c52a96448ca033dacbe21acbd61156a6ee19589c1e558af7fb54"},"motivation":"Mobile agent benchmarks increasingly rely on LLM-based judges to evaluate task completion, yet the reliability of these judges on mobile agent trajectories remains largely unexamined.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.11434","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_formstruct-bench_6e003e69","familyId":"bmf_b049023d4081","name":"FormStruct-Bench","oneLine":"The paper introduces FormStruct-Bench, an evaluation dataset for table-form document structure recognition, but no official links to the benchmark artifacts are provided.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.10396","pdf":"https://arxiv.org/pdf/2608.10396","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.10396"},"evidence":{"snippet":"We introduce FormStruct-Bench, a hierarchical and diagnostic benchmark that evaluates table-form document structure recognition at both the document level and progressively finer component levels, allowing aggregate performance to be traced back to specific structural failure modes.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.10396"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"The paper introduces FormStruct-Bench, an evaluation dataset for table-form document structure recognition, but no official links to the benchmark artifacts are provided.","whyItMatters":"An evaluation gap exists between holistic document outputs and fine-grained structural scores, but the lack of public benchmark access limits its practical utility.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"03bdec5f287b3a644b8c4bedb02000fe36036d6d5c753cc269d2c2615317d78b"},"motivation":"Transforming table-form documents into machine-processable records requires recovering not only their visible content but also the multilevel structure that organizes it.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.10396","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_faithformbench_8ab938df","familyId":"bmf_e9cebcae140a","name":"FaithformBench","oneLine":"A benchmark for evaluating faithfulness of mathematical chain-of-thought autoformalisation, using perturbed reasoning steps to test validity and invalidity preservation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.10916","pdf":"https://arxiv.org/pdf/2608.10916","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.10916"},"evidence":{"snippet":"To address these limitations, we propose a new benchmark for AF faithfulness that is cheap to apply, sound under weak assumptions, and assesses both positive and negative examples.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.10916"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark for evaluating faithfulness of mathematical chain-of-thought autoformalisation, using perturbed reasoning steps to test validity and invalidity preservation.","whyItMatters":"Addresses the need for sound and cheap evaluation of autoformalisation faithfulness, revealing sycophancy in current systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7fad17407de799e1a4cca6650362a0a22e1a56c63e10f265e61d35b348ac7f9b"},"motivation":"Autoformalisation (AF) systems map natural language reasoning steps into formal statements in a proof assistant such as Lean.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.10916","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_dsagentbench_96674f61","familyId":"bmf_3b3386cca544","name":"DSAgentBench","oneLine":"DSAgentBench evaluates language agents on end-to-end data-science workflows in real computer environments. It comprises 275 tasks spanning data wrangling, exploration, modeling, visualization, and validation, with deterministic evaluators verifying analytical correctness, visual outputs, and model performance.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.10366","pdf":"https://arxiv.org/pdf/2608.10366","project":null,"code":"https://github.com/vis-nlp/DSAgentBench","data":null,"hfPaper":"https://huggingface.co/papers/2608.10366"},"evidence":{"snippet":"We introduce DSAgentBench, the first benchmark to evaluate whether agents can automate full data-science workflows inside real computer environments.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":10,"hfDailySubmittedAt":"2026-08-12T00:00:00.000Z","githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.10366"},"ranking":{"30d":{"score":64,"rank":40,"coverage":0.85,"confidence":"High"},"90d":{"score":50,"rank":195,"coverage":0.7,"confidence":"Medium"}},"description":"DSAgentBench evaluates language agents on end-to-end data-science workflows in real computer environments. It comprises 275 tasks spanning data wrangling, exploration, modeling, visualization, and validation, with deterministic evaluators verifying analytical correctness, visual outputs, and model performance.","whyItMatters":"Existing benchmarks lack real-computer interaction and fail to capture the multi-stage, multi-tool nature of data-science practice. DSAgentBench provides a realistic environment for assessing whether agents can automate complete workflows, highlighting a significant capability gap and offering a foundation for developing grounded, verifiable autonomous agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"13a48bd9c8ed99ab691727a6ba073787497b8648c31139f631ca072d3c8ca472"},"motivation":"Real-world data science involves long-horizon workflows that span data wrangling, exploration, modeling, visualization, and validation, and require coordinated use of tools such as notebooks, IDEs, terminals, browsers, and databases within real operating environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.10366","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"vis-nlp","organizationType":"academic-lab","sourceUrl":"https://github.com/vis-nlp/DSAgentBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_criticalscm-bench_de361f05","familyId":"bmf_4b271b812478","name":"CriticalSCM-Bench","oneLine":"CriticalSCM-Bench v1 is a synthetic supply-chain benchmark with causal ground truth, paired factual/counterfactual rollouts, and an explicit net-value objective. It evaluates intervention ranking policies across semiconductor, critical-material, and digital-infrastructure archetypes under partial and delayed information, with stress tests on fidelity, timing, cost, and held-out disruptions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.LG"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.11154","pdf":"https://arxiv.org/pdf/2608.11154","project":null,"code":"https://github.com/dyshang/dacri-criticalscm-bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.11154"},"evidence":{"snippet":"We present CriticalSCM-Bench v1, a controlled synthetic benchmark with causal ground truth, paired factual/counterfactual rollouts, and an explicit net-value objective.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.11154"},"ranking":{"30d":{"score":34,"rank":131,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":354,"coverage":0.55,"confidence":"Low"}},"description":"CriticalSCM-Bench v1 is a synthetic supply-chain benchmark with causal ground truth, paired factual/counterfactual rollouts, and an explicit net-value objective. It evaluates intervention ranking policies across semiconductor, critical-material, and digital-infrastructure archetypes under partial and delayed information, with stress tests on fidelity, timing, cost, and held-out disruptions.","whyItMatters":"The benchmark addresses the gap between detection and decision by quantifying recoverable net value of interventions, helping choose between adaptive ranking and simpler structural policies based on domain conditions and out-of-distribution robustness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d0f1674313ce4334d618dc923948904ab6122956d8f3cd02b28d0e09f79ae244"},"motivation":"Detecting or attributing a supply-chain disruption is not the same as selecting the intervention that maximizes recoverable net value.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"presentation at the International Conference on Electrical, Computer, Communications and Mechatronics Engineering (ICECC","evidence":"Accepted for presentation at the International Conference on Electrical, Computer, Communications and Mechatronics Engineering (ICECCME 2026), 15--17 October 2026, Bali, Indonesia. 5 tables; no figures. Benchmark and code: https://github.com/dyshang/dacri-criticalscm-bench","evidenceUrl":"https://arxiv.org/abs/2608.11154","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"presentation at the International Conference on Electrical, Computer, Communications and Mechatronics Engineering (ICECC","reviewStatus":"accepted","decisionRaw":"Accepted for presentation at the International Conference on Electrical, Computer, Communications and Mechatronics Engineering (ICECCME 2026), 15--17 October 2026, Bali, Indonesia. 5 tables; no figures. Benchmark and code: https://github.com/dyshang/dacri-criticalscm-bench","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.11154","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted for presentation at the International Conference on Electrical, Computer, Communications and Mechatronics Engineering (ICECCME 2026), 15--17 October 2026, Bali, Indonesia. 5 tables; no figures. Benchmark and code: https://github.com/dyshang/dacri-criticalscm-bench","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_chartography_26c993c6","familyId":"bmf_ac7027e150a2","name":"Chartography","oneLine":"Chartography evaluates vision-language models on 100 chart-understanding tasks sourced from professional practice, with questions authored by domain experts and triple-verified. The benchmark uses pass@1 accuracy as the scoring metric across 30 frontier model configurations.","area":"Vision & 3D","applicationDomains":["Industrial & Engineering","Finance & Economics"],"primaryDomain":"Industrial & Engineering","industrySectors":["Manufacturing","Financial Services"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.10677","pdf":"https://arxiv.org/pdf/2608.10677","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.10677"},"evidence":{"snippet":"We introduce Chartography, a benchmark of 100 tasks that pair charts drawn from professional practice, in domain-specific formats that standard chart benchmarks rarely include, with questions written by professionals who read these charts for a living and independently verified by three additional experts.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.10677"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Chartography evaluates vision-language models on 100 chart-understanding tasks sourced from professional practice, with questions authored by domain experts and triple-verified. The benchmark uses pass@1 accuracy as the scoring metric across 30 frontier model configurations.","whyItMatters":"Existing chart benchmarks are saturated and skewed toward simple formats, leaving a gap in measuring performance on realistic, domain-specific charts. Chartography provides a harder, expert-validated test that can differentiate model capabilities in high-stakes professional contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"596c35b8826016fed1a9d257e3e85883bbd76f1636a65e7969490e6c422c1f50"},"motivation":"Professionals across medicine, engineering, finance, manufacturing, and the sciences often make consequential decisions from charts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"2nd Workshop on Benchmarking Evidence-Aligned Multimodal Reasoning (BEAM 2), ECCV 2026","evidence":"16 pages, 5 figures, 5 tables. Accepted at the 2nd Workshop on Benchmarking Evidence-Aligned Multimodal Reasoning (BEAM 2), ECCV 2026","evidenceUrl":"https://arxiv.org/abs/2608.10677","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"2nd Workshop on Benchmarking Evidence-Aligned Multimodal Reasoning (BEAM 2), ECCV 2026","reviewStatus":"accepted","decisionRaw":"16 pages, 5 figures, 5 tables. Accepted at the 2nd Workshop on Benchmarking Evidence-Aligned Multimodal Reasoning (BEAM 2), ECCV 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.10677","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"16 pages, 5 figures, 5 tables. Accepted at the 2nd Workshop on Benchmarking Evidence-Aligned Multimodal Reasoning (BEAM 2), ECCV 2026","level":"author-claim"}]}],"publishers":[{"name":"Chartography Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.10677","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"cross-domain"},{"id":"bm_capprobe_99438ec2","familyId":"bmf_1dc0b6750669","name":"CapProbe","oneLine":"CapProbe is a full-scene dense QA benchmark for evaluating detailed image captions from Vision-Language Models. It decomposes images into semantic regions and generates multiple-choice questions across 10 semantic categories, with a language judge answering from captions. The benchmark comprises 346 images, 1,868 regions, and 25,650 questions.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.11074","pdf":"https://arxiv.org/pdf/2608.11074","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.11074"},"evidence":{"snippet":"We introduce CapProbe, a full-scene dense QA benchmark that turns detailed caption evaluation into region-aligned factual checking.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.11074"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CapProbe is a full-scene dense QA benchmark for evaluating detailed image captions from Vision-Language Models. It decomposes images into semantic regions and generates multiple-choice questions across 10 semantic categories, with a language judge answering from captions. The benchmark comprises 346 images, 1,868 regions, and 25,650 questions.","whyItMatters":"Existing metrics for detailed caption evaluation struggle to verify dense factual claims. CapProbe addresses this by region-aligned factual checking with dense QA, offering a cost-effective protocol that reduces open-ended scoring bias and reveals coverage gaps and trade-offs across models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0b0ea05a9983cd82fa70f568d7715455979e4e4beca8085fd1dea1bd6387fc1f"},"motivation":"Evaluating detailed image captions from Vision-Language Models (VLMs) requires going beyond surface-level semantic similarity.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.11074","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_videovibe_8a828619","familyId":"bmf_58daaf002b88","name":"VideoVIBE","oneLine":"Evaluates video-grounded diagnostic understanding of one-shot website generation. Approximately 1.7K Video QA instances from 6,338 verified failures across semantic-logical, visual-motion, structural-temporal, and functional categories. Scoring is based on model accuracy in multi-agent system V2Lens compared against baseline Video MLLMs.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09573","pdf":"https://arxiv.org/pdf/2608.09573","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09573"},"evidence":{"snippet":"We introduce VideoVIBE, a video-grounded benchmark that transforms human-operated webpage recordings into fine-grained diagnostic tasks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09573"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates video-grounded diagnostic understanding of one-shot website generation. Approximately 1.7K Video QA instances from 6,338 verified failures across semantic-logical, visual-motion, structural-temporal, and functional categories. Scoring is based on model accuracy in multi-agent system V2Lens compared against baseline Video MLLMs.","whyItMatters":"Existing benchmarks often score isolated artifacts or final outcomes, lacking diagnostic insight. This benchmark provides a repeatable protocol to assess failure modes in generated webpages, enabling targeted model improvements for interactive website generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d727bd676b33f4f6bb5b9b1547b0e4dad2f8f56b84ca9144f2ef0d368c35acd7"},"motivation":"Natural-language-driven \"vibe coding\" enables the one-shot generation of visually rich and interactive web applications, yet reliable assessment of their quality has not kept pace.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09573","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_usertoolbench_99873053","familyId":"bmf_2846551f77dd","name":"UserToolBench","oneLine":"UserToolBench evaluates personalized decision making in tool-use LLMs through inference of latent user preferences, clarification need, and user-aligned tool-call trajectories, built from privacy-sanitized interaction traces with 10 user profiles, 36 tool sets, and 1,065 turns.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.10042","pdf":"https://arxiv.org/pdf/2608.10042","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.10042"},"evidence":{"snippet":"We introduce UserToolBench , a benchmark for personalized decision making in tool-use LLMs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.10042"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"UserToolBench evaluates personalized decision making in tool-use LLMs through inference of latent user preferences, clarification need, and user-aligned tool-call trajectories, built from privacy-sanitized interaction traces with 10 user profiles, 36 tool sets, and 1,065 turns.","whyItMatters":"Current personalization benchmarks focus on surface-level style imitation or response personalization, not whether models make correct decisions for the user. UserToolBench addresses this gap by emphasizing decision quality, which is critical for real-world delegation tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e8ada43b31b6212f475fea541069db0033ca0cfc6d6846cc0ebc3a7528f1eb01"},"motivation":"Tool-use LLMs are increasingly asked to act on users' behalf, but existing benchmarks usually focus on profile recall, style imitation, generic tool use, or response-level personalization.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.10042","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_tcs-bench_6efad642","familyId":"bmf_4a4db0639414","name":"TCS-BENCH","oneLine":"Evaluates LLMs on research-level theoretical computer science proof generation, using theorem-proving tasks from papers at STOC, FOCS, and SODA. Provides context for self-contained proofs and uses a verification agent to check correctness.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09538","pdf":"https://arxiv.org/pdf/2608.09538","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09538"},"evidence":{"snippet":"We introduce TCS-Bench, a benchmark for evaluating Large Language Models (LLMs) on research-level Theoretical Computer Science (TCS) proof generation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09538"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates LLMs on research-level theoretical computer science proof generation, using theorem-proving tasks from papers at STOC, FOCS, and SODA. Provides context for self-contained proofs and uses a verification agent to check correctness.","whyItMatters":"Fills a gap in evaluating LLMs on advanced formal reasoning in theoretical computer science, offering a structured protocol with a verification agent that aligns closely with expert judgment, supporting reliable model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"060db29575e76c4ae56b1dc4f7b4cc70fbeaea26c0107ca4cbda855375d97d94"},"motivation":"We introduce TCS-Bench, a benchmark for evaluating Large Language Models (LLMs) on research-level Theoretical Computer Science (TCS) proof generation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09538","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TCS-Bench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.09538","role":"benchmark-publisher"}],"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_taf-med_ece2193a","familyId":"bmf_437ad9931b00","name":"TAF-MED","oneLine":"TAF-MED is a physician-reviewed benchmark of 500 fixed three-turn medical safety scenarios, evaluating LLM responses for unsafe guidance across multi-turn dialogues.","area":"Safety & Trustworthiness","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.10258","pdf":"https://arxiv.org/pdf/2608.10258","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.10258"},"evidence":{"snippet":"We introduce TAF-MED, a physician-reviewed benchmark of 500 fixed three-turn scenarios, and evaluate eight LLMs across 4,000 conversations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.10258"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"TAF-MED is a physician-reviewed benchmark of 500 fixed three-turn medical safety scenarios, evaluating LLM responses for unsafe guidance across multi-turn dialogues.","whyItMatters":"Addresses the evaluation gap where first-turn safety is an incomplete proxy for conversational safety persistence, providing a protocol for assessing model behavior across complete dialogue trajectories in medical contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c67df0ae4961ae1cee8f27b4ea1848b457e8ef6298a8975347d93513da9dab4a"},"motivation":"Large language models (LLMs) increasingly provide conversational health information that may influence treatment decisions, yet existing benchmarks do not isolate whether medication-safety boundaries persist across follow-ups after explicit self-treatment intent.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.10258","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_swe-bench_6afbfb26","familyId":"bmf_7e98773a0ce5","name":"SWE-Bench ProMax","oneLine":"A multilingual code refactoring benchmark with 170 instances drawn from real commits across seven programming languages. Evaluates AI agents on large-scale refactoring tasks averaging 11.4 modified files and 261.6 lines of code, using manually curated issue descriptions and test suites.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation"],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09802","pdf":"https://arxiv.org/pdf/2608.09802","project":null,"code":null,"data":"https://huggingface.co/datasets/swe-bench-promax/SWE-Bench-ProMax","hfPaper":"https://huggingface.co/papers/2608.09802"},"evidence":{"snippet":"We introduce SWE-Bench ProMax, an expert-curated, multilingual code refactoring benchmark of 170 instances drawn from real commits across seven programming languages (Python, Java, TypeScript, Go, C, C++, and Rust).","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":133,"hfDailySubmittedAt":"2026-08-11T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":344,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2608.09802"},"ranking":{"30d":{"score":69,"rank":32,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":5,"datasetRankPopulation":20},"90d":{"score":64,"rank":107,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":15,"datasetRankPopulation":51}},"description":"A multilingual code refactoring benchmark with 170 instances drawn from real commits across seven programming languages. Evaluates AI agents on large-scale refactoring tasks averaging 11.4 modified files and 261.6 lines of code, using manually curated issue descriptions and test suites.","whyItMatters":"Existing software engineering benchmarks face saturation and quality issues, with flawed tests and training data leakage. This benchmark provides a more challenging and realistic refactoring task set with rigorous curation, offering a robust measure of agent capability for long-horizon coding tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fa4c4a74da32976391c428ed62af680b43ba9fd7cdf6d17b40d422e01bcd079f"},"motivation":"As AI coding agents take on increasingly complex, long-horizon software engineering tasks, existing benchmarks are rapidly saturating and their evaluation quality has come under serious scrutiny: a recent audit found that nearly 60% of unsolved SWE-bench Verified instances contain flawed tests -- either overly narrow tests that reject correct solutions or overly broad tests that check unstated requirements -- and th…","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09802","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SWE-Bench-ProMax Team","organizationType":"academic-lab","sourceUrl":"https://huggingface.co/datasets/swe-bench-promax/SWE-Bench-ProMax","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_sci-vbench_8bc62acf","familyId":"bmf_063001d7adb8","name":"Sci-VBench","oneLine":"Sci-VBench evaluates text-to-video generation across 1,253 expert-annotated examples in 60 scientific subjects. It requires temporally rich videos demonstrating scientific reasoning and knowledge-grounded synthesis. Scoring covers four dimensions: prompt grounding, scientific correctness, spatiotemporal consistency, and low-level perceptual fidelity, using a rubric-based protocol with MLLM judges.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Factuality"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09873","pdf":"https://arxiv.org/pdf/2608.09873","project":null,"code":"https://github.com/sci-vbench/sci-vbench","data":null,"hfPaper":"https://huggingface.co/papers/2608.09873"},"evidence":{"snippet":"We introduce Sci-VBench, a comprehensive benchmark for evaluating knowledge- and reasoning-intensive video generation across scientific domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":29,"hfDailySubmittedAt":"2026-08-11T00:00:00.000Z","githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09873"},"ranking":{"30d":{"score":72,"rank":27,"coverage":0.85,"confidence":"High"},"90d":{"score":55,"rank":170,"coverage":0.7,"confidence":"Medium"}},"description":"Sci-VBench evaluates text-to-video generation across 1,253 expert-annotated examples in 60 scientific subjects. It requires temporally rich videos demonstrating scientific reasoning and knowledge-grounded synthesis. Scoring covers four dimensions: prompt grounding, scientific correctness, spatiotemporal consistency, and low-level perceptual fidelity, using a rubric-based protocol with MLLM judges.","whyItMatters":"Existing video generation benchmarks focus on surface realism, leaving scientific and causal correctness unmeasured. Sci-VBench provides a public protocol and open dataset to compare models on knowledge-intensive generation, revealing gaps between visual quality and reliable scientific dynamics.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0bea81b4ff8609c221e7040b2848b4fe21dbaa5d03563ab582c5f638e731ee58"},"motivation":"We introduce Sci-VBench, a comprehensive benchmark for evaluating knowledge- and reasoning-intensive video generation across scientific domains.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09873","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_safescenereason_c358bce3","familyId":"bmf_7f963029c8c3","name":"SafeSceneReason","oneLine":"SafeSceneReason evaluates multimodal industrial-safety reasoning with 123,695 question-answer pairs covering compliance, hazard interaction, accident mechanisms, and prevention recommendations across scene-centric and report-centric pipelines.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Factuality"],"topics":["Multimodal","Safety","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09230","pdf":"https://arxiv.org/pdf/2608.09230","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09230"},"evidence":{"snippet":"We introduce SafeSceneReason, a multimodal industrial-safety reasoning benchmark and companion training corpus that connects workplace scenes with knowledge from occupational accident investigations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09230"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SafeSceneReason evaluates multimodal industrial-safety reasoning with 123,695 question-answer pairs covering compliance, hazard interaction, accident mechanisms, and prevention recommendations across scene-centric and report-centric pipelines.","whyItMatters":"Existing safety datasets test perception or isolated violations, leaving a gap in evidence-grounded reasoning. This benchmark differentiates model competence in comparative, technical, and multi-evidence reasoning, offering decision value for industrial safety applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2581022515208fffc9aeb20f9f67a4dd65299dad78eb9320defe140fad1729e2"},"motivation":"Industrial-safety understanding requires more than detecting workers, equipment, and personal protective equipment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09230","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_pragmatch_5cfd2fc5","familyId":"bmf_c5b11c599fa8","name":"PragMatch","oneLine":"PragMatch is a controlled set of 3,000 image-text pairs derived from MMSD2.0 for studying pragmatic incongruity in multimodal sarcasm detection, with original sarcastic examples and constructed literal and hard-negative pairs.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09772","pdf":"https://arxiv.org/pdf/2608.09772","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09772"},"evidence":{"snippet":"We introduce PragMatch, a controlled benchmark of 3,000 image-text pairs derived from MMSD2.0, including original sarcastic examples and constructed literal and hard-negative pairs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09772"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PragMatch is a controlled set of 3,000 image-text pairs derived from MMSD2.0 for studying pragmatic incongruity in multimodal sarcasm detection, with original sarcastic examples and constructed literal and hard-negative pairs.","whyItMatters":"This resource helps investigate whether large vision-language models rely on superficial cues instead of genuine reasoning in multimodal sarcasm, highlighting practical limitations in model evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"02b3858c0d5e91311baa7320036df25cb670bac676f468442886c33674339fa7"},"motivation":"Large Vision-Language Models (LVLMs) have demonstrated strong performance on multimodal benchmarks, yet it remains unclear whether they genuinely reason about relationships between images and text or rely on superficial correlations, known as shortcut learning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09772","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_nba-streaming_a4cd00b8","familyId":"bmf_c0442d23bb74","name":"NBA_Streaming","oneLine":"NBA_Streaming is a benchmark for online fine-grained basketball commentary generation, containing 307.5 hours of broadcasts and approximately 35K temporally aligned events with annotations of event boundaries, player identities, fine-grained actions, event chains, and natural-language commentary.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09200","pdf":"https://arxiv.org/pdf/2608.09200","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09200"},"evidence":{"snippet":"To address these limitations, we introduce NBA_Streaming, a large-scale benchmark for online fine-grained basketball commentary generation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09200"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"NBA_Streaming is a benchmark for online fine-grained basketball commentary generation, containing 307.5 hours of broadcasts and approximately 35K temporally aligned events with annotations of event boundaries, player identities, fine-grained actions, event chains, and natural-language commentary.","whyItMatters":"It addresses the evaluation gap in streaming sports video understanding and generation, enabling unified assessment of event localization, response reliability, factual grounding, and commentary quality under causal constraints.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0c7eb2c6d9c31457a37d3d9f57524a8312faa1faa4466562e40fdcfd95de5d42"},"motivation":"Live basketball commentary generation requires determining when an event is sufficiently observable and describing it before subsequent events unfold.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09200","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mmarch_3c23535d","familyId":"bmf_cde8c1453aed","name":"MMArch","oneLine":"MMArch is a benchmark for multimodal reasoning in architecture and civil engineering, spanning ten subdomains and built from figures in peer-reviewed papers. It contains 1,212 short-answer items requiring perception, principle identification, and application.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09281","pdf":"https://arxiv.org/pdf/2608.09281","project":"https://dcx-swjtu.github.io/MMArch/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09281"},"evidence":{"snippet":"We introduce MMArch, a benchmark for architecture and civil engineering spanning ten subdomains and built entirely from figures in peer-reviewed papers.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09281"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MMArch is a benchmark for multimodal reasoning in architecture and civil engineering, spanning ten subdomains and built from figures in peer-reviewed papers. It contains 1,212 short-answer items requiring perception, principle identification, and application.","whyItMatters":"Existing benchmarks test drawing recognition or information extraction, but MMArch evaluates whether models can combine distributed visual evidence with engineering principles, filling a gap in multimodal reasoning evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1791fc684d00be2eaa297e2ddec1284cf0fe1b3277486b536ab94ee02070430f"},"motivation":"Multimodal large language models (MLLMs) perform strongly on engineering imagery, yet existing benchmarks mostly test drawing recognition, information extraction, or compliance checking, leaving open whether models can combine distributed visual evidence with engineering principles to reach a conclusion.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09281","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"dcx-swjtu","organizationType":"academic-lab","sourceUrl":"https://dcx-swjtu.github.io/MMArch/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_madbench_d9b9325a","familyId":"bmf_14d0ddd00f54","name":"MADBench","oneLine":"MADBench is a benchmark for modality-aware audio deepfake detection, treating speech and environmental audio as distinct components. It evaluates detectors across independently manipulated forgery sources.","area":"Vision & 3D","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.SD"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.09593","pdf":"https://arxiv.org/pdf/2608.09593","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09593"},"evidence":{"snippet":"We introduce MADBench, the first benchmark that treats speech and environmental audio as distinct acoustic components, enabling component-aware evaluation of audio deepfake detection across independently manipulated forgery sources.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09593"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MADBench is a benchmark for modality-aware audio deepfake detection, treating speech and environmental audio as distinct components. It evaluates detectors across independently manipulated forgery sources.","whyItMatters":"It addresses the gap in audio deepfake detection by distinguishing speech and background audio, which have different generative mechanisms and artifact profiles, enabling component-aware evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"496597daaf8b11a6e0c4362f80da2c471a192314db625b5d7c4bb8a0b92f83ee"},"motivation":"Recent advances in speech synthesis and audio generation have made high-fidelity acoustic forgery low-cost and difficult to attribute, enabling a realistic attack scenario in which speech and background audio are independently manipulated over otherwise authentic video.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09593","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_lexkairos_261e303a","familyId":"bmf_453e64d655a7","name":"LexKairos","oneLine":"LexKairos is a benchmark for evaluating the temporal capabilities of LLMs in the Chinese legal context, covering statutory temporal knowledge, case temporal modeling, and statute-case temporal reasoning across nine sub-tasks.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Factuality"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09106","pdf":"https://arxiv.org/pdf/2608.09106","project":null,"code":"https://github.com/thunlp/LexKairos","data":null,"hfPaper":"https://huggingface.co/papers/2608.09106"},"evidence":{"snippet":"To address this gap, we propose LexKairos, a comprehensive benchmark for evaluating the temporal capabilities of LLMs in the Chinese legal context across three dimensions: statutory temporal knowledge, case temporal modeling, and statute-case temporal reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09106"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LexKairos is a benchmark for evaluating the temporal capabilities of LLMs in the Chinese legal context, covering statutory temporal knowledge, case temporal modeling, and statute-case temporal reasoning across nine sub-tasks.","whyItMatters":"Legal temporal capabilities are underexplored in existing legal AI benchmarks; LexKairos provides a public tool for evaluating time-sensitive legal reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d2eba5efeddac2ec01fea4127da96229a994117c9f0c30617482e507d044eda4"},"motivation":"Large language models (LLMs) have demonstrated strong performance across a wide range of legal tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09106","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"THUNLP","organizationType":"academic-lab","sourceUrl":"https://github.com/thunlp/LexKairos","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_guardian-crawler_0a6465c9","familyId":"bmf_bbf29cd625e1","name":"Guardian Crawler","oneLine":"Guardian Crawler is a retrieval-first testbed for knowledge discovery over synthetic web-like corpora, combining BM25 retrieval with reranking and constrained generation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval","Factuality"],"topics":["cs.IR"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.08994","pdf":"https://arxiv.org/pdf/2608.08994","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.08994"},"evidence":{"snippet":"We present Guardian Crawler, a reproducible retrieval-first testbed for controlled experiments on knowledge discovery and evidence-grounded summarization over synthetic web-like corpora.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08994"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Guardian Crawler is a retrieval-first testbed for knowledge discovery over synthetic web-like corpora, combining BM25 retrieval with reranking and constrained generation.","whyItMatters":"It provides a controlled environment for experiments on evidence-grounded summarization in noisy domains.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"56b51d3eb5ed6b35547e3a1ec438410d1ded469e82b2c9d748b8ac7e8b73c039"},"motivation":"Retrieving relevant evidence from noisy web data is challenging, particularly in sensitive domains containing incomplete reports, heterogeneous language, and irrelevant content.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"as a Short Paper at KDIR 2026 (International Conference on Knowledge Discovery and Information Retrieval)","evidence":"8 pages, 2 figures. Accepted as a Short Paper at KDIR 2026 (International Conference on Knowledge Discovery and Information Retrieval)","evidenceUrl":"https://arxiv.org/abs/2608.08994","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"as a Short Paper at KDIR 2026 (International Conference on Knowledge Discovery and Information Retrieval)","reviewStatus":"accepted","decisionRaw":"8 pages, 2 figures. Accepted as a Short Paper at KDIR 2026 (International Conference on Knowledge Discovery and Information Retrieval)","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.08994","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"8 pages, 2 figures. Accepted as a Short Paper at KDIR 2026 (International Conference on Knowledge Discovery and Information Retrieval)","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_evo-bench_a13a17e9","familyId":"bmf_d555691c91f5","name":"Evo-Bench","oneLine":"Evo-Bench is a benchmark for evaluating whether language models can autonomously improve their agent harness, with 608 harness-sensitive tasks across Search, Office, and General domains, fixed policy model, and resource budget.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09096","pdf":"https://arxiv.org/pdf/2608.09096","project":null,"code":"https://github.com/RUCAIBox/Evo-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.09096"},"evidence":{"snippet":"To address these challenges, we introduce Evo-Bench, the first benchmark designed to evaluate models' intrinsic harness-evolving capabilities across Search, Office, and General agent domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":18,"hfDailySubmittedAt":"2026-08-11T00:00:00.000Z","githubStars":9,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09096"},"ranking":{"30d":{"score":76,"rank":17,"coverage":0.85,"confidence":"High"},"90d":{"score":62,"rank":121,"coverage":0.7,"confidence":"Medium"}},"description":"Evo-Bench is a benchmark for evaluating whether language models can autonomously improve their agent harness, with 608 harness-sensitive tasks across Search, Office, and General domains, fixed policy model, and resource budget.","whyItMatters":"It isolates harness evolution from base model strength, addressing a gap in evaluating agent self-improvement capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"84ac3298bdc134f6072c77d653b047219d85d1f68801c26089d65708bb1e5d62"},"motivation":"Large Language Models (LLMs) have driven rapid progress in autonomous agents, yet standard evaluations remain confined to static task solving.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09096","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"RUCAIBox","organizationType":"academic-lab","sourceUrl":"https://github.com/RUCAIBox/Evo-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_elbench_6c33e0fa","familyId":"bmf_3e9cf94fe4c5","name":"ELBench","oneLine":"ELBench is a benchmark for education-facing large language models, evaluating General Capability, Safety and Trustworthiness, Basic Education, and High-Level Cultivation under a common protocol with 2,939 items.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Inspectable","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09548","pdf":"https://arxiv.org/pdf/2608.09548","project":null,"code":null,"data":"https://huggingface.co/datasets/ZeroLoss-Lab/ELBench","hfPaper":"https://huggingface.co/papers/2608.09548"},"evidence":{"snippet":"We introduce ELBench, the first benchmark to evaluate all four requirements (General Capability, Safety and Trustworthiness, Basic Education, and High-Level Cultivation) on the same models under a common protocol, combining curated public sources with newly synthesized safety and cultivation data.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":54,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2608.09548"},"ranking":{"30d":{"score":47,"rank":83,"coverage":0.15,"confidence":"Low","datasetDownloadRank":15,"datasetRankPopulation":20},"90d":{"score":41,"rank":264,"coverage":0.3,"confidence":"Low","datasetDownloadRank":41,"datasetRankPopulation":51}},"description":"ELBench is a benchmark for education-facing large language models, evaluating General Capability, Safety and Trustworthiness, Basic Education, and High-Level Cultivation under a common protocol with 2,939 items.","whyItMatters":"It fills the gap of integrated evaluation for education-facing models, which require accuracy, safety, instructional usefulness, and pedagogical alignment simultaneously.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"efb65a84f9e941745f3e6784878ce52e4a604593d16968449b1d92c1dde37136"},"motivation":"Large language models are increasingly deployed in education as tutors, teaching assistants, and content generators.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09548","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ZeroLoss-Lab","organizationType":"academic-lab","sourceUrl":"https://huggingface.co/datasets/ZeroLoss-Lab/ELBench","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_comboshoppingbench_78836e04","familyId":"bmf_9c63aab8e01d","name":"ComboShoppingBench","oneLine":"ComboShoppingBench is a benchmark for agentic basket shopping with coupons, evaluating LLM agents on constructing feasible baskets with budget and coupon constraints in a simulated environment.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09282","pdf":"https://arxiv.org/pdf/2608.09282","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09282"},"evidence":{"snippet":"We introduce ComboShoppingBench, an agentic shopping benchmark for open-ended yet verifiable basket construction in a simulated commerce and takeout environment.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09282"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ComboShoppingBench is a benchmark for agentic basket shopping with coupons, evaluating LLM agents on constructing feasible baskets with budget and coupon constraints in a simulated environment.","whyItMatters":"It addresses the gap in evaluating agents for combinatorial shopping tasks, which require joint reasoning about compatibility, availability, and constraints.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"18268737539c82cec70a4fc624a8fc044c7cf86e22c84acf8730489620fd3b7c"},"motivation":"Real-world shopping often requires constructing a basket of complementary items rather than retrieving a single product.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09282","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_chronostate_2f5e6d2f","familyId":"bmf_ba0e55d44cb0","name":"ChronoState","oneLine":"ChronoState evaluates whether a frozen language model can compose hidden elapsed-time scalars with symbolic task state to select temporal actions, using forced-choice accuracy under direct supervision.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09124","pdf":"https://arxiv.org/pdf/2608.09124","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09124"},"evidence":{"snippet":"We introduce ChronoState, a compositional temporal-state benchmark in which symbolic state appears in the prompt, elapsed seconds tau are supplied through a hidden chronometric-injection channel, and the model selects a forced-choice temporal action.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09124"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ChronoState evaluates whether a frozen language model can compose hidden elapsed-time scalars with symbolic task state to select temporal actions, using forced-choice accuracy under direct supervision.","whyItMatters":"The benchmark investigates a narrow mechanism for injecting time information into LLMs, but the results do not generalize broadly and the setup is primarily for probing one architectural variant.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fe4b7db4c7f15f1f812314917070aee3df98627475821ab3584cb5d3ac38ca89"},"motivation":"Temporal decisions in language-model systems often depend on both symbolic task state and elapsed wall-clock time, such as cache expiration, job completion, quota resets, deadlines, or stale sessions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09124","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cadengbench_ba1f84a8","familyId":"bmf_3e931326d51e","name":"CADEngBench","oneLine":"CADEngBench evaluates parametric CAD part generation and editing through B-Rep validity, engineering checks, parameter perturbation, and linear-static FEA, plus assembly reasoning via joint retrieval, grounding, and kinematic verification.","area":"Science & Engineering","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Geometric reasoning"],"topics":["CAD","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09296","pdf":"https://arxiv.org/pdf/2608.09296","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09296"},"evidence":{"snippet":"We present CADEngBench, a two-track benchmark for these capabilities.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09296"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"CADEngBench evaluates parametric CAD part generation and editing through B-Rep validity, engineering checks, parameter perturbation, and linear-static FEA, plus assembly reasoning via joint retrieval, grounding, and kinematic verification.","whyItMatters":"It addresses the gap in CAD evaluation by focusing on engineering behavior rather than visual appearance, providing a structured test for design validity, functional editing, and assembly correctness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c2109b05e810c3ab40d8f744de90c7a2e1b2c291d4fed0a2371855efb762ad1e"},"motivation":"A CAD model is not engineering-grade merely because it looks correct.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09296","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_avalon-tom-bench_a3c88250","familyId":"bmf_370b127bbad7","name":"Avalon-ToM-Bench","oneLine":"Avalon-ToM-Bench evaluates fine-grained theory of mind in LLMs using a 2x2 taxonomy of epistemic/motivational reasoning crossed with inference/action, via human-crafted perspective-constrained queries from The Resistance: Avalon.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-10","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.09638","pdf":"https://arxiv.org/pdf/2608.09638","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09638"},"evidence":{"snippet":"We present Avalon-ToM-Bench, a fine-grained benchmark that operationalizes ToM through the asymmetric-information mechanics of The Resistance: Avalon.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09638"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Avalon-ToM-Bench evaluates fine-grained theory of mind in LLMs using a 2x2 taxonomy of epistemic/motivational reasoning crossed with inference/action, via human-crafted perspective-constrained queries from The Resistance: Avalon.","whyItMatters":"It provides a diagnostic decomposition of ToM abilities, distinguishing reasoning, expression, and policy, and offers insights into training and inference interventions for improving social reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e7e97550c1421ec4e82d853d9f652f9020d620a4fa0cf9319826d7e8a2b35e6c"},"motivation":"Theory of Mind (ToM) is essential for agent interactions, yet existing evaluations either rely on static scenarios that oversimplify mental-state reasoning or interactive settings that provide limited diagnostic insight.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09638","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_skillsafe-bench_e266705a","familyId":"bmf_1ffd6ecd5782","name":"SkillSafe-Bench","oneLine":"SkillSafe-Bench evaluates skill-merged LLMs on static refusal, adaptive jailbreak robustness, and capability retention using a two-judge AND rule, across multiple open-weight bases and attack types.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Robustness"],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.08542","pdf":"https://arxiv.org/pdf/2608.08542","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.08542"},"evidence":{"snippet":"We introduce SkillSafe-Bench, a controlled benchmark that scores skill-merged models on static refusal, adaptive jailbreak robustness, and capability retention under a conservative two-judge AND rule.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08542"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SkillSafe-Bench evaluates skill-merged LLMs on static refusal, adaptive jailbreak robustness, and capability retention using a two-judge AND rule, across multiple open-weight bases and attack types.","whyItMatters":"It exposes that static safety does not predict robustness to adaptive attacks, providing a more accurate safety evaluation for model merging and guiding safer merging practices.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cde271257299f16d1e2889b842f9edf7caa9647aa300443ef945774ed634650d"},"motivation":"Model merging has become the default way to give an aligned language model new skills without retraining: a practitioner folds task vectors from math, code, or domain specialists into a safety-aligned base using task arithmetic, TIES, or DARE.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.08542","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_skillreason-bench_12813801","familyId":"bmf_71b30f2ba1c9","name":"SkillReason-Bench","oneLine":"SkillReason-Bench is a retrieval benchmark with 3,729 queries and 61,228 skills across nine domains, but the paper focuses on proposing a retrieval method (SkillReason) rather than the benchmark itself.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Information retrieval"],"topics":["Agents","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.08640","pdf":"https://arxiv.org/pdf/2608.08640","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.08640"},"evidence":{"snippet":"To address this gap, we introduce SkillReason-Bench, a large-scale cross-domain benchmark containing 3,729 queries and a retrieval corpus of 61,228 skills spanning nine domains.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08640"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SkillReason-Bench is a retrieval benchmark with 3,729 queries and 61,228 skills across nine domains, but the paper focuses on proposing a retrieval method (SkillReason) rather than the benchmark itself.","whyItMatters":"The benchmark serves as a testbed for the proposed method and is compared to existing benchmarks, but the primary contribution is the method, not a standalone reusable benchmark.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"21f818c80bbcc303d0410a059f0296a48bab3d92a36efb6edb9bc0964b74ec47"},"motivation":"Large language model agents increasingly rely on reusable skills to extend their capabilities beyond parametric knowl- edge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.08640","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_plugineval_1642b04d","familyId":"bmf_604048e8cd8d","name":"PluginEval","oneLine":"PluginEval evaluates tool routing in LLMs via three decision types (missed, spurious, parameter errors) across difficulty levels, using deterministic validation and real API execution for reliable signals.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.08700","pdf":"https://arxiv.org/pdf/2608.08700","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.08700"},"evidence":{"snippet":"In this paper, we introduce PluginEval, a benchmark constructed through a two-stage framework that systematically mitigates these limitations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08700"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PluginEval evaluates tool routing in LLMs via three decision types (missed, spurious, parameter errors) across difficulty levels, using deterministic validation and real API execution for reliable signals.","whyItMatters":"It overcomes limitations of power-law data distributions and unvalidated LLM judgments, offering a diagnostic error profile for function calling and enabling more reliable agent evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e7eaa41ec8defa91b5c9ad920ac29c0c8093ad0dd1c09846df78e9190ed9dbc7"},"motivation":"Reliable evaluation of tool routing is critical as Large Language Models increasingly operate as autonomous agents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.08700","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_math-vision-diagrams_40787688","familyId":"bmf_54b9a73db7dc","name":"Math-Vision Diagrams","oneLine":"Math-Vision Diagrams evaluates LLMs on mathematical diagram generation from text, covering both text-to-code and text-to-image paradigms, with a curated subset of 2,920 competition problems and multiple metrics.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.08964","pdf":"https://arxiv.org/pdf/2608.08964","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.08964"},"evidence":{"snippet":"We introduce Math-Vision Diagrams, the first benchmark specifically designed to evaluate LLMs on mathematical diagram generation, and the first to assess text-to-code and text-to-image generation paradigms together in a single unified setting, agnostic of the underlying coding lan- guage or model type.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08964"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Math-Vision Diagrams evaluates LLMs on mathematical diagram generation from text, covering both text-to-code and text-to-image paradigms, with a curated subset of 2,920 competition problems and multiple metrics.","whyItMatters":"It fills the gap in standardized evaluation of math diagram generation, enabling comparison across paradigms and models, and provides a comprehensive benchmark for this emerging capability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a93b713c407a583ff127d3b2a3e591b713c9f6ce56c2dd934bb71782bfbf916b"},"motivation":"The generation of mathematically precise diagrams from tex- tual prompts has emerged as a critical yet underexplored capability of Large Language Models (LLMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICANN 2026","evidence":"Accepted at ICANN 2026","evidenceUrl":"https://arxiv.org/abs/2608.08964","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICANN 2026","reviewStatus":"accepted","decisionRaw":"Accepted at ICANN 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.08964","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at ICANN 2026","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception","Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_fitaqa_e0a97bca","familyId":"bmf_3b8db6654700","name":"FitAQA","oneLine":"FitAQA evaluates fitness action quality assessment in MLLMs across perception, judgement, and temporal grounding tasks, using a unified taxonomy of 38 form errors in six dimensions over 30 exercises.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.08736","pdf":"https://arxiv.org/pdf/2608.08736","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.08736"},"evidence":{"snippet":"We introduce FitAQA, a systematic benchmark for evaluating MLLMs in fitness AQA, containing 2,219 videos and 5,512 QA instances across 30 bodyweight exercises.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08736"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FitAQA evaluates fitness action quality assessment in MLLMs across perception, judgement, and temporal grounding tasks, using a unified taxonomy of 38 form errors in six dimensions over 30 exercises.","whyItMatters":"It provides a systematic benchmark for fitness AQA, enabling assessment of MLLMs' ability to perceive and reason about exercise quality, and identifies visual perception as a key bottleneck.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"21de7b802bd8ae885b3add2d9aa4da55c9bd3bef1b3c613a3199e2b920b327ad"},"motivation":"Fitness Action Quality Assessment (AQA) is important for intelligent sports training, yet the capabilities of Multimodal Large Language Models (MLLMs) in this setting remain underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.08736","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_doc2db-bench_2f955ada","familyId":"bmf_589f2dd26eba","name":"Doc2DB-Bench","oneLine":"Doc2DB-Bench evaluates document-to-database construction, converting long heterogeneous documents into normalized relational databases with entity identities, keys, cross-table links, and integrity constraints. It includes 203 document instances across 42 schemas and 7 domains, with fine-grained capability annotations for intra-table extraction and inter-table reasoning.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.08459","pdf":"https://arxiv.org/pdf/2608.08459","project":null,"code":"https://github.com/SetonLiang/Doc2DB-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.08459"},"evidence":{"snippet":"We introduce Doc2DB-Bench, a benchmark for Document-to-Database construction, containing 203 long-document instances across 42 schemas and seven domain groups, with 117 entity tables, 132 relationship tables, 7,341 rows, and 41,935 cells.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08459"},"ranking":{"30d":{"score":50,"rank":71,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":280,"coverage":0.55,"confidence":"Low"}},"description":"Doc2DB-Bench evaluates document-to-database construction, converting long heterogeneous documents into normalized relational databases with entity identities, keys, cross-table links, and integrity constraints. It includes 203 document instances across 42 schemas and 7 domains, with fine-grained capability annotations for intra-table extraction and inter-table reasoning.","whyItMatters":"Existing document-to-table benchmarks overlook relational database requirements such as normalization, entity resolution, and cross-table consistency. Doc2DB-Bench addresses this gap by providing a testbed for assessing LLM-based systems in realistic database construction, supporting analytics, compliance, and decision-making applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1a30e1a1267c2ee86ee536f5d0089979d7489042c158d140a7974e1c3b2d2840"},"motivation":"Practical AI systems increasingly need to turn long, heterogeneous documents into queryable relational databases, not isolated spreadsheets.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.08459","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Doc2DB-Bench Team","organizationType":"academic-lab","sourceUrl":"https://github.com/SetonLiang/Doc2DB-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cap_c96b1f01","familyId":"bmf_45e84635bc48","name":"CAP","oneLine":"CAP evaluates cross-site browser agents on 420 tasks across 108 real websites and 24 domains. Tasks require complex UI interactions and visual perception, with scoring via an agent-as-a-judge framework.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.08392","pdf":"https://arxiv.org/pdf/2608.08392","project":"https://warriorxu0302.github.io/CAP-Bench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.08392"},"evidence":{"snippet":"We introduce CAP, a scalable benchmark for evaluating browser agents on cross-site, human-like web tasks that require non-trivial UI interactions and visual understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08392"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CAP evaluates cross-site browser agents on 420 tasks across 108 real websites and 24 domains. Tasks require complex UI interactions and visual perception, with scoring via an agent-as-a-judge framework.","whyItMatters":"CAP addresses the gap in browser agent evaluation by focusing on cross-site workflows and perception-heavy interactions, which are common in real-world browsing. It provides fine-grained diagnostics to identify bottlenecks in current agents, aiding targeted improvements.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2477e8d117b2c7ccb623acd131c8c26a2ed1022dbd26a5b62590a12880799d86"},"motivation":"Large language models are increasingly deployed as autonomous agents that interact with the web through browsers.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"COLM 2026","evidence":"Accepted to COLM 2026. Project page: https://warriorxu0302.github.io/CAP-Bench/","evidenceUrl":"https://arxiv.org/abs/2608.08392","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"COLM 2026","reviewStatus":"accepted","decisionRaw":"Accepted to COLM 2026. Project page: https://warriorxu0302.github.io/CAP-Bench/","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.08392","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to COLM 2026. Project page: https://warriorxu0302.github.io/CAP-Bench/","level":"author-claim"}]}],"publishers":[{"name":"CAP-Bench Project","organizationType":"academic-lab","sourceUrl":"https://warriorxu0302.github.io/CAP-Bench/","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_360cityarena_2a62b93d","familyId":"bmf_28f4efc5217e","name":"360CityArena","oneLine":"360CityArena evaluates embodied agents in a photorealistic virtual urban environment built from 360-degree video of Tokyo's Akihabara district. It includes 175 tasks across environment understanding, path reasoning, and spatial reasoning, testing localization, landmark search, path planning, and relational spatial reasoning. Scoring uses mean evaluation score with varied success criteria.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.08814","pdf":"https://arxiv.org/pdf/2608.08814","project":"https://360mm-team.github.io/360CityArena/","code":"https://github.com/360MM-Team/360CityArena","data":null,"hfPaper":"https://huggingface.co/papers/2608.08814"},"evidence":{"snippet":"We present 360CityArena, a benchmark for evaluating the urban exploration capabilities of embodied agents within a photorealistic environment constructed from 360-degree videos.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":10,"hfDailySubmittedAt":"2026-08-12T00:00:00.000Z","githubStars":18,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08814"},"ranking":{"30d":{"score":74,"rank":21,"coverage":0.85,"confidence":"High"},"90d":{"score":66,"rank":97,"coverage":0.7,"confidence":"Medium"}},"description":"360CityArena evaluates embodied agents in a photorealistic virtual urban environment built from 360-degree video of Tokyo's Akihabara district. It includes 175 tasks across environment understanding, path reasoning, and spatial reasoning, testing localization, landmark search, path planning, and relational spatial reasoning. Scoring uses mean evaluation score with varied success criteria.","whyItMatters":"Existing outdoor navigation benchmarks lack photorealism or complexity, leaving a gap between simulated and real-world urban conditions. This benchmark provides a repeatable protocol with released code, tasks, and evaluation scripts, enabling comparative assessment of embodied agents' city-scale navigation and reasoning abilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2170a2e8e48844a0ccd97d4d848ea0c771da141cf2082f6ca914a5eab7d982f3"},"motivation":"We present 360CityArena, a benchmark for evaluating the urban exploration capabilities of embodied agents within a photorealistic environment constructed from 360-degree videos.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.08814","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"360MM Team","organizationType":"academic-lab","sourceUrl":"https://github.com/360MM-Team/360CityArena","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_vqabench_22600a6b","familyId":"bmf_16fc6c20e67a","name":"VQABench","oneLine":"Evaluates 12 image preprocessing techniques for cloud VLM-based visual question answering across 3 VQA datasets and 4 commercial models, measuring accuracy, cost, and latency.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07861","pdf":"https://arxiv.org/pdf/2608.07861","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07861"},"evidence":{"snippet":"To fill this gap, we present VQABench, the first systematic benchmark that treats client-side input preprocessing as a controlled variable for cloud-VLM-based VQA.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07861"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates 12 image preprocessing techniques for cloud VLM-based visual question answering across 3 VQA datasets and 4 commercial models, measuring accuracy, cost, and latency.","whyItMatters":"Assesses the impact of client-side preprocessing on cost-quality trade-offs for offloaded VQA inference, offering practical guidance for system design.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"51bfe3bd7cb7dc9126d5da07383cad13b5e77a25a0fe6cdc4271a7b688d807cb"},"motivation":"Vision-language models (VLMs) are becoming a practical backend for mobile visual question answering (VQA) systems, enabling smartphones and smart glasses to answer users' questions about the physical world.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07861","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_telemetrysuffbench_4e3a485c","familyId":"bmf_384333dc77aa","name":"TelemetrySuffBench","oneLine":"TelemetrySuffBench evaluates models on fault-origin diagnosis in agent telemetry, using controlled multi-component traces with delayed-binding faults, paired coarse views, seven-factor telemetry masks, and exact-equal ambiguous origin pairs.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-08","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.07899","pdf":"https://arxiv.org/pdf/2608.07899","project":"https://anonymous.4open.science/r/TelemetrySuffBench-E635/README.md","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07899"},"evidence":{"snippet":"We introduce TelemetrySuffBench, a controlled benchmark that separates failure detection, fault-origin localization, and safe abstention under insufficient evidence.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07899"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TelemetrySuffBench evaluates models on fault-origin diagnosis in agent telemetry, using controlled multi-component traces with delayed-binding faults, paired coarse views, seven-factor telemetry masks, and exact-equal ambiguous origin pairs.","whyItMatters":"It addresses the evaluation gap in diagnosing failure origins from agent telemetry, showing that full telemetry yields high localization accuracy but coarse views preserve detection while limiting localization, highlighting the need for explicit decision-to-provenance links and abstention safeguards.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"999e1e8453fe5c641369afe105d68244f23808c3b6efe18cc7cf2ad49a9c3889"},"motivation":"Agent systems increasingly expose execution traces, yet telemetry that reveals a failure may still be inadequate for identifying where that failure originated.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07899","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Anonymous","organizationType":"community","sourceUrl":"https://anonymous.4open.science/r/TelemetrySuffBench-E635/README.md","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_surgwmbench_1b6538d4","familyId":"bmf_12d0da504604","name":"SurgWMBench","oneLine":"Evaluates surgical world models on short-horizon instrument motion prediction and dynamics stability from intraoperative image sequences, focusing on geometric accuracy and temporal coherence.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning"],"topics":["cs.RO"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-08","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.08070","pdf":"https://arxiv.org/pdf/2608.08070","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.08070"},"evidence":{"snippet":"In this paper, we introduce SurgWMBench, a vision-based benchmark for short-horizon surgical motion planning and dynamics prediction.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08070"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates surgical world models on short-horizon instrument motion prediction and dynamics stability from intraoperative image sequences, focusing on geometric accuracy and temporal coherence.","whyItMatters":"Provides a standardized protocol for motion-centric evaluation in surgical world models, addressing the lack of public datasets and metrics aligned with instrument motion planning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"fc729d568ad9455b830b5d8d22eba3661b1edafd4c99cc0f656e87c2b0c7c35b"},"motivation":"Reliable surgical planning requires models that move beyond recognizing the current surgical step or imitating expert demonstrations, and instead anticipate how instrument motion reshapes subsequent operative states.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.08070","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SurgWMBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.08070","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_surakshaeval_6928de7a","familyId":"bmf_9fb058d84496","name":"SurakshaEval","oneLine":"SurakshaEval evaluates the safety of LLMs across ten major Indian languages and English, using human-written prompts covering generic and region-specific scenarios, with a structured scoring protocol.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07862","pdf":"https://arxiv.org/pdf/2608.07862","project":null,"code":"https://github.com/debobanerjee/SurakshaEval","data":null,"hfPaper":"https://huggingface.co/papers/2608.07862"},"evidence":{"snippet":"To address this gap, we introduce SurakshaEval, a novel safety benchmark composed of human-written prompts spanning real-world scenarios, explicitly designed for ten major Indian languages - Assamese, Bengali, Gujarati, Hindi, Kannada, Malayalam, Marathi, Punjabi, Tamil, and Telugu, along with English.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07862"},"ranking":{"30d":{"score":24,"rank":142,"coverage":0.85,"confidence":"High"},"90d":{"score":25,"rank":386,"coverage":0.7,"confidence":"Medium"}},"description":"SurakshaEval evaluates the safety of LLMs across ten major Indian languages and English, using human-written prompts covering generic and region-specific scenarios, with a structured scoring protocol.","whyItMatters":"Existing safety benchmarks are largely English-centric, leaving a gap for multilingual and culturally grounded safety assessment. SurakshaEval offers a public protocol to benchmark LLM safety in Indian languages, aiding deployment in diverse linguistic contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e1c84909b838af12685cfca2c2dbae3628bfdb6f910f490a3a3bc8da9212d43e"},"motivation":"Existing safety evaluation datasets for large language models (LLMs) predominantly focus on English and Western contexts, often overlooking the linguistic diversity and culturally grounded safety risks present in other languages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07862","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SurakshaEval Benchmark Team","organizationType":"academic-lab","sourceUrl":"https://github.com/debobanerjee/SurakshaEval","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_oblivion_83c0aaa0","familyId":"bmf_ea841fbbb09f","name":"OBLIVION","oneLine":"Evaluates operational skill unlearning in deployed agents across 88 attack episodes, measuring attack success rate and impact-weighted exposure after workflow-level defenses.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.08264","pdf":"https://arxiv.org/pdf/2608.08264","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.08264"},"evidence":{"snippet":"We introduce OBLIVION, a controlled benchmark and defense harness for revoked-skill resurrection.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08264"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates operational skill unlearning in deployed agents across 88 attack episodes, measuring attack success rate and impact-weighted exposure after workflow-level defenses.","whyItMatters":"Introduces a measurable benchmark for a new safety problem: preventing agents from rebuilding revoked skills, supporting workflow-level evaluation beyond parameter forgetting.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"c80ff3f8bf8837012becc8b8c6cfb5f4dbd569fe5211db4f27ada2a8a8565a31"},"motivation":"Large language model agents are becoming operational interfaces to files, memories, registries, and external tools.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.08264","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"OBLIVION Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.08264","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_ncp-bench_83c91412","familyId":"bmf_b98627ff590e","name":"NCP-Bench","oneLine":"NCP-Bench evaluates narrative commitment preservation in interactive narratives. It provides 100 movie-synopsis-based environments with structured narrative specifications and an automatic evaluator that checks fact, commitment, and trajectory consistency across narrator responses to player actions.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-08","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.08160","pdf":"https://arxiv.org/pdf/2608.08160","project":null,"code":"https://github.com/yingpengma/NCP-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.08160"},"evidence":{"snippet":"We introduce NCP-Bench, a benchmark of 100 narrative environments derived from movie synopses.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":29,"hfDailySubmittedAt":"2026-08-13T00:00:00.000Z","githubStars":29,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08160"},"ranking":{"30d":{"score":84,"rank":7,"coverage":0.85,"confidence":"High"},"90d":{"score":72,"rank":65,"coverage":0.7,"confidence":"Medium"}},"description":"NCP-Bench evaluates narrative commitment preservation in interactive narratives. It provides 100 movie-synopsis-based environments with structured narrative specifications and an automatic evaluator that checks fact, commitment, and trajectory consistency across narrator responses to player actions.","whyItMatters":"The benchmark fills a gap in evaluating long-horizon logical consistency of LLM-driven interactive narrators, offering a repeatable protocol to compare models on narrative integrity under adversarial user interventions. This supports practical selection of models for interactive storytelling applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"61b0d5cc322af2e5dcd34aabfcf5d9257813a42e0dbee2d2254ad8a895921b72"},"motivation":"The rapid advancement of Large Language Models (LLMs) is revolutionizing AI for Games by enabling open-ended and fluid interactive storytelling.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026","evidence":"Accepted by ICML 2026","evidenceUrl":"https://arxiv.org/abs/2608.08160","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026","reviewStatus":"accepted","decisionRaw":"Accepted by ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.08160","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by ICML 2026","level":"author-claim"}]}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_lingjing_2ca7c6e7","familyId":"bmf_da300d8a728c","name":"Lingjing","oneLine":"Lingjing is a simulation testbed for heterogeneous multi-agent embodied tasks in urban environments, with a Gym-like interface and support for natural-language missions. It includes engine-based evaluations and attribution-ready replays.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-08","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.08045","pdf":"https://arxiv.org/pdf/2608.08045","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.08045"},"evidence":{"snippet":"Lingjing provides a unified testbed that enables reproducible end-to-end evaluation and systematic failure diagnosis in urban multi-agent embodied intelligence.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.08045"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Lingjing is a simulation testbed for heterogeneous multi-agent embodied tasks in urban environments, with a Gym-like interface and support for natural-language missions. It includes engine-based evaluations and attribution-ready replays.","whyItMatters":"If published, it could support reproducible evaluation of multi-agent urban embodied intelligence, but without a public release path, its standalone value is unverified.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"569f247cfcb5899a04416f64e8dba1b1935bd5286911092874fadbbdc287430b"},"motivation":"Urban embodied intelligence requires coordination among heterogeneous agents (e.g., UAVs, ground robots, and autonomous vehicles) in dynamic cities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.08045","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_libad_b1abf215","familyId":"bmf_dac7b1dda460","name":"LIBAD","oneLine":"LIBAD is a multimodal anomaly detection dataset from Li-ion battery electrode manufacturing, providing aligned visible-light and X-ray radiography images. It includes benchmarks of methods under inline-compatible settings.","area":"Multimodal","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Manufacturing"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07958","pdf":"https://arxiv.org/pdf/2608.07958","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07958"},"evidence":{"snippet":"We introduce LIBAD, the first multimodal anomaly detection benchmark for Li-ion battery electrode manufacturing.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07958"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"LIBAD is a multimodal anomaly detection dataset from Li-ion battery electrode manufacturing, providing aligned visible-light and X-ray radiography images. It includes benchmarks of methods under inline-compatible settings.","whyItMatters":"If available, it could enable evaluation of anomaly detection in continuous manufacturing with weakly correlated modalities, but without a release path, its standalone value is unverified.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8851abd4cae76729fd2f845d26f84411a7d1dc9d2c85a70abbf0557ed4563887"},"motivation":"Multimodal industrial anomaly detection has largely focused on discrete products using strongly correlated RGB and 3D observations, leaving continuous process manufacturing and weakly correlated sensing modalities underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07958","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_texfix-bench_827bc253","familyId":"bmf_2ca72336c626","name":"TeXFix-Bench","oneLine":"TeXFix-Bench evaluates LLM-based full-source document repair across LaTeX, Typst, and Markdown. It provides 10,437 repair instances derived from 743 openly licensed seeds, with a fixed zero-shot protocol and provider-pinned routing.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07617","pdf":"https://arxiv.org/pdf/2608.07617","project":"https://doi.org/10.5281/zenodo.21831797","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07617"},"evidence":{"snippet":"We present TeXFix-Bench, a multi-format benchmark for LLM-based full-source document repair grounded in a mined fault taxonomy.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07617"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TeXFix-Bench evaluates LLM-based full-source document repair across LaTeX, Typst, and Markdown. It provides 10,437 repair instances derived from 743 openly licensed seeds, with a fixed zero-shot protocol and provider-pinned routing.","whyItMatters":"The benchmark fills a gap in document-repair evaluation by grounding faults in a mined taxonomy, offering a reproducible protocol to compare models on compile success and content restoration. This supports practical selection of models for document repair tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"acc301d5ec7116053aeaab1da5a6b9174b9ee3c12137db81658a85a793be5a30"},"motivation":"Scientific and technical writing depends on markup sources that must compile: LaTeX, Typst, and Markdown pipelines fail on missing delimiters, mismatched environments, broken imports, or package conflicts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07617","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Zenodo","organizationType":"community","sourceUrl":"https://doi.org/10.5281/zenodo.21831797","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_skysealand_02e2341b","familyId":"bmf_9cf9f27a7978","name":"SkySeaLand","oneLine":"SkySeaLand is a satellite object detection dataset with 1,307 high-resolution images and 19,101 bounding boxes across four classes. It provides COCO and YOLO annotations, a common split, and COCO metrics for detector evaluation.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.07382","pdf":"https://arxiv.org/pdf/2608.07382","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07382"},"evidence":{"snippet":"SkySeaLand provides a compact benchmark for mixed land--maritime transportation detection, while SkyDet establishes a documented low-footprint reference rather than a state-of-the-art accuracy claim.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07382"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SkySeaLand is a satellite object detection dataset with 1,307 high-resolution images and 19,101 bounding boxes across four classes. It provides COCO and YOLO annotations, a common split, and COCO metrics for detector evaluation.","whyItMatters":"The benchmark addresses a gap in wide-format satellite imagery detection, offering a compact dataset with a fixed protocol to compare detectors under standard metrics, supporting practical model selection for transportation monitoring.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7daac9c2572581f44e854a045803e215d0c3b2717b75e62d2df5212cb3522fd1"},"motivation":"Satellite object detection is challenged by small targets and wide-format scenes that lose detail under standard square-input resizing.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07382","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_regiondet_ecd9208e","familyId":"bmf_57bf14a770e8","name":"RegionDet","oneLine":"RegionDet is a benchmark for region detection with eight categories, using COCO-style bounding-box annotations and evaluation protocols. It aims to extend object detection to region-level targets.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.06850","pdf":"https://arxiv.org/pdf/2608.06850","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.06850"},"evidence":{"snippet":"To address this gap, we introduce Region Detection, a task that extends conventional object detection beyond object instances, and construct RegionDet, a benchmark for region target localization.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06850"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RegionDet is a benchmark for region detection with eight categories, using COCO-style bounding-box annotations and evaluation protocols. It aims to extend object detection to region-level targets.","whyItMatters":"If published, it could support evaluation of region-level detection, but without a public release path, its standalone value is unverified.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"51a81b99f11669864ebb64e31d3955dd541862c727f48d049025ef47b5edcefb"},"motivation":"Object detection is a fundamental task in computer vision and has achieved remarkable progress on standard benchmarks by localizing discrete and well-bounded object instances.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06850","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_littraceqa_955ebe65","familyId":"bmf_c0a9eaf07d4f","name":"LitTraceQA","oneLine":"LitTraceQA evaluates literature-grounded question answering over scientific papers, requiring systems to return paper IDs, evidence locations, and answers in multiple formats. The public split includes 55 examples with gold annotations.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07370","pdf":"https://arxiv.org/pdf/2608.07370","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07370"},"evidence":{"snippet":"We present LitTraceQA, a benchmark for literature-grounded question answering over scientific papers.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07370"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LitTraceQA evaluates literature-grounded question answering over scientific papers, requiring systems to return paper IDs, evidence locations, and answers in multiple formats. The public split includes 55 examples with gold annotations.","whyItMatters":"The benchmark fills a gap in evaluating verifiable scientific QA, separating retrieval, grounding, and answer accuracy, providing a reusable testbed for systems that produce evidence-backed responses.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d95068159e4bc1b52bc5a43f15d065ca9cd34b00b931a053d1205a82e885ffd5"},"motivation":"Scientific literature is increasingly used as a knowledge source for language models, retrieval-augmented generation systems, and research assistants, but answering research questions from papers requires more than fluent generation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07370","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_heatcast_ee7ee561","familyId":"bmf_e7e61f16dc25","name":"HeatCast","oneLine":"HeatCast is a benchmark for monthly Land Surface Temperature forecasting at 30m resolution across 124 U.S. cities. It provides Landsat-based tiles, fixed temporal split, LCZ-stratified metrics, and a reference evaluation harness.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07640","pdf":"https://arxiv.org/pdf/2608.07640","project":"https://doi.org/10.57967/hf/9889","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07640"},"evidence":{"snippet":"We introduce HeatCast, a Landsat-based benchmark for monthly LST forecasting across 124 U.S.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07640"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HeatCast is a benchmark for monthly Land Surface Temperature forecasting at 30m resolution across 124 U.S. cities. It provides Landsat-based tiles, fixed temporal split, LCZ-stratified metrics, and a reference evaluation harness.","whyItMatters":"The benchmark addresses a gap in neighborhood-scale LST forecasting, offering a large-scale dataset with standardized metrics to compare forecasting models across diverse urban environments, supporting practical applications in urban heat monitoring.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3bba44789bcc04b64d3d0eb0e83bd9abe163c16015e710f969ee0fda9c0363e0"},"motivation":"Land Surface Temperature (LST) is a widely used satellite-derived measure of urban surface heat, but there is no shared benchmark for forecasting it at 30 m.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07640","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_harnesssafe_835e5e2c","familyId":"bmf_bde338e32b63","name":"HarnessSafe","oneLine":"HarnessSafe evaluates safety of agent harnesses through 328 executable cases across seven persistent-carrier families, using trace-based evaluation to track attack chains from entry to potential violation.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.06984","pdf":"https://arxiv.org/pdf/2608.06984","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.06984"},"evidence":{"snippet":"To this end, we present HarnessSafe, a benchmark comprising 328 executable cases across seven persistent-carrier families and evaluated on most mainstream agent harnesses.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06984"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HarnessSafe evaluates safety of agent harnesses through 328 executable cases across seven persistent-carrier families, using trace-based evaluation to track attack chains from entry to potential violation.","whyItMatters":"Addresses the lack of benchmarks covering multiple persistent carriers and providing trace-level analysis, offering a more nuanced assessment of safety risks in agent systems compared to end-to-end success rates.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9625ca8c248cc9847bc16bbed502a8d2ee8162f6614bd9356d9aa38c78b00e4f"},"motivation":"Modern agent harnesses persist state across tasks and sessions through persistent carriers like memory, skills, tools, and shared artifacts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06984","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_graphverse_5c59831b","familyId":"bmf_52691295a1bc","name":"GraphVerse","oneLine":"GraphVerse evaluates multimodal large language models on visual graph reasoning, covering perception, reasoning, and text-based graph reasoning in single and paired image settings, with process-sensitive scoring.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.06769","pdf":"https://arxiv.org/pdf/2608.06769","project":null,"code":"https://github.com/sunyuanfu/GraphVerse","data":null,"hfPaper":"https://huggingface.co/papers/2608.06769"},"evidence":{"snippet":"To bridge the gap, we introduce GraphVerse, a unified benchmark that jointly evaluates perception, visual reasoning, and text-based graph reasoning in MLLMs under both single-image and paired-image settings.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06769"},"ranking":{"30d":{"score":34,"rank":132,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":355,"coverage":0.55,"confidence":"Low"}},"description":"GraphVerse evaluates multimodal large language models on visual graph reasoning, covering perception, reasoning, and text-based graph reasoning in single and paired image settings, with process-sensitive scoring.","whyItMatters":"Provides a unified benchmark for visual graph reasoning that goes beyond answer-only metrics, addressing gaps in existing evaluations and enabling assessment of reasoning quality.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"915f9e7086ce709ee6882ffcfd511aee15232a67b33142505e69d462bc62bd94"},"motivation":"Recent Multimodal Large Language Models (MLLMs) have achieved remarkable progress across diverse vision-language tasks, creating an urgent need for more challenging benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06769","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GraphVerse Team","organizationType":"academic-lab","sourceUrl":"https://github.com/sunyuanfu/GraphVerse","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_generative-embedding-benchmark_a087748d","familyId":"bmf_e7d9564105e1","name":"Generative Embedding Benchmark","oneLine":"GEB evaluates embeddings by measuring answer-relevant content recoverable by a decoder, using a visual question-answering dataset with development and test splits, and scoring answer quality.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.06972","pdf":"https://arxiv.org/pdf/2608.06972","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.06972"},"evidence":{"snippet":"To address this gap, we introduce the Generative Embedding Benchmark (GEB), in which a decoder answers questions using only a frozen embedding and question text, without access to the original image or intermediate visual features.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06972"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"GEB evaluates embeddings by measuring answer-relevant content recoverable by a decoder, using a visual question-answering dataset with development and test splits, and scoring answer quality.","whyItMatters":"Fills a gap by measuring generative information in embeddings, which is not captured by separability-based benchmarks, providing insight into information preservation for downstream generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d69187a6af41b75f5dc5ec62382306abe11b189f588a69c2178e6ba24b19450f"},"motivation":"Embeddings have emerged as a standard representational interface linking foundation models with downstream systems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06972","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_finrank_f391afeb","familyId":"bmf_defe6e205daf","name":"FinRank","oneLine":"FinRank evaluates passage retrieval, reranking, and hard-negative discrimination over SEC filings, with 1185 question-answer records and curated hard negatives, scoring recall and accuracy.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":["Information retrieval"],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.07400","pdf":"https://arxiv.org/pdf/2608.07400","project":null,"code":"https://github.com/datanxt/FinRank","data":null,"hfPaper":"https://huggingface.co/papers/2608.07400"},"evidence":{"snippet":"The benchmark contains 1185 manually authored question-answer records over the 10-K and 10-Q filings of 22 companies.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07400"},"ranking":{"30d":{"score":50,"rank":72,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":281,"coverage":0.55,"confidence":"Low"}},"description":"FinRank evaluates passage retrieval, reranking, and hard-negative discrimination over SEC filings, with 1185 question-answer records and curated hard negatives, scoring recall and accuracy.","whyItMatters":"Targets provenance-sensitive financial QA, addressing the challenge of grounding answers in correct evidence, which is critical for compliance and trust in financial applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4aa060e159017ea16c219efb693d48e1d6173862880157a80bb18385e19e9292"},"motivation":"Financial question answering is typically evaluated by answer correctness, yet in SEC filings a plausible and even numerically correct answer can be grounded in the wrong evidence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07400","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FinRank Team","organizationType":"academic-lab","sourceUrl":"https://github.com/datanxt/FinRank","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"specific"},{"id":"bm_clinicare-bench_6faa931e","familyId":"bmf_23f50f44501d","name":"CliniCARE-Bench","oneLine":"CliniCARE-Bench evaluates clinical agents on retrospective audit tasks over longitudinal EHR data, measuring verdict accuracy, evidence grounding, process adherence, calibrated abstention, reliability, and efficiency.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07796","pdf":"https://arxiv.org/pdf/2608.07796","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07796"},"evidence":{"snippet":"We introduce CliniCARE-Bench (Clinical Calibrated Audit of Medical Reasoning in EHR), a benchmark for retrospective clinical audit: 25 clinician-validated scenarios instantiated as 750 patient-specific cases over real-patient-derived MIMIC-IV data.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07796"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CliniCARE-Bench evaluates clinical agents on retrospective audit tasks over longitudinal EHR data, measuring verdict accuracy, evidence grounding, process adherence, calibrated abstention, reliability, and efficiency.","whyItMatters":"Provides a deployment-oriented benchmark for clinical agents, assessing not just accuracy but also defensibility and calibration, which are essential for trustworthy clinical decision support.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"71096404cea83d695392b31e689eaf23aa9d62f96a486bcbb0535b76d507d261"},"motivation":"Large language models perform strongly on medical knowledge benchmarks, but reliable clinical deployment requires agents to conduct defensible investigations over heterogeneous, longitudinal records: determining what evidence is needed, retrieving and reconciling structured and free-text data, grounding conclusions in verifiable evidence, and deferring cases that cannot be resolved reliably.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07796","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_binjudgebench_a8dd4ae5","familyId":"bmf_03c435b933bf","name":"BinJudgeBench","oneLine":"BinJudgeBench evaluates LLM-as-a-Judge for human-oriented binary reverse engineering, covering function name recovery, code summarization, and decompilation optimization, with correlation to human judgment as the metric.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07038","pdf":"https://arxiv.org/pdf/2608.07038","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07038"},"evidence":{"snippet":"We introduce BinJudgeBench, the first expert-annotated, reference-free evaluation benchmark based on multi-dimensional human judgment, where LLM-as-a-Judge achieves an average correlation of 63.20\\% with human judgment, outperforming traditional automated metrics at 35.04\\%.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07038"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"BinJudgeBench evaluates LLM-as-a-Judge for human-oriented binary reverse engineering, covering function name recovery, code summarization, and decompilation optimization, with correlation to human judgment as the metric.","whyItMatters":"Addresses the challenge of scalable evaluation for HOBRE, providing a reference-free alternative that correlates with human judgment better than traditional metrics.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4b4ab63edcb579c568a05f86c94df18cc102240ad5090bbf7dac85476beb9f54"},"motivation":"Human-Oriented Binary Reverse Engineering (HOBRE) aims to transform decompiled pseudocode into a more human-friendly representation, thereby reducing the cognitive burden of reverse analysis and improving efficiency.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"41st IEEE/ACM International Conference on Automated Software Engineering (ASE 2026)","evidence":"Accepted by the 41st IEEE/ACM International Conference on Automated Software Engineering (ASE 2026)","evidenceUrl":"https://arxiv.org/abs/2608.07038","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"41st IEEE/ACM International Conference on Automated Software Engineering (ASE 2026)","reviewStatus":"accepted","decisionRaw":"Accepted by the 41st IEEE/ACM International Conference on Automated Software Engineering (ASE 2026)","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.07038","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by the 41st IEEE/ACM International Conference on Automated Software Engineering (ASE 2026)","level":"author-claim"}]}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_archegraph_38554c04","familyId":"bmf_9e9576facb0b","name":"ArchEGraph","oneLine":"ArchEGraph evaluates geometry-topology-physics aligned building energy modeling through graph reconstruction and topology-informed load prediction tasks, with standardized protocols and generalization experiments.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.06772","pdf":"https://arxiv.org/pdf/2608.06772","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.06772"},"evidence":{"snippet":"We present ArchEGraph, a large-scale benchmark dataset that represents buildings as heterogeneous graphs with aligned geometry, topology, weather, and zone-level thermal loads.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06772"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"ArchEGraph evaluates geometry-topology-physics aligned building energy modeling through graph reconstruction and topology-informed load prediction tasks, with standardized protocols and generalization experiments.","whyItMatters":"Provides a large-scale dataset and benchmark tasks for building energy modeling, enabling development and evaluation of surrogate models that couple geometry, topology, and physics.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6d821a6450bba6ab7d634139e126804552f6ab3c729aa12c2b085ba13218dc04"},"motivation":"Accurate estimation of building energy use is essential for achieving carbon neutral and sustainable buildings.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06772","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_xeworld_c6f125ea","familyId":"bmf_0e0ce96d0a58","name":"XEWorld","oneLine":"XEWorld is a testbed for evaluating cross-embodiment generalization of action-conditioned world models, but it is primarily a research probe without a defined public benchmark protocol.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.05799","pdf":"https://arxiv.org/pdf/2608.05799","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.05799"},"evidence":{"snippet":"To answer whether a model can faithfully render a robot it has never seen, we introduce XEWorld, a controlled cross-embodiment testbed for world models that isolates embodiments by evaluating held-out robots within physically identical scenes.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.05799"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"XEWorld is a testbed for evaluating cross-embodiment generalization of action-conditioned world models, but it is primarily a research probe without a defined public benchmark protocol.","whyItMatters":"The study highlights limitations in current world models, but the evaluation setup is tied to the paper's analysis and lacks a standalone comparison path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dd575afd8c5009f71301c6debc14c44f0e2cc26970a8b8e9478ab69b2b9bec67"},"motivation":"Action-conditioned world models are promising learned simulators for robotic manipulation, yet evaluating them exclusively on training robots fails to reveal whether they capture physical dynamics or merely memorize visual patterns.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.05799","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_tradeverse_c4be7871","familyId":"bmf_29d08cd7c2a4","name":"TradeVerse","oneLine":"TradeVerse evaluates LLMs on longitudinal political trade negotiation understanding using reconstructed minutes of 1170 WTO meetings across 5 groups and 89 product groups, with three tasks: predicting HS codes, identifying responding countries, and generating final statements.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.06549","pdf":"https://arxiv.org/pdf/2608.06549","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.06549"},"evidence":{"snippet":"We introduce TradeVerse, a benchmark built from the World Trade Organisation (WTO) specific trade concerns, where member states challenge one another and exchange arguments over multiple rounds, sometimes for years.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06549"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TradeVerse evaluates LLMs on longitudinal political trade negotiation understanding using reconstructed minutes of 1170 WTO meetings across 5 groups and 89 product groups, with three tasks: predicting HS codes, identifying responding countries, and generating final statements.","whyItMatters":"It addresses the evaluation gap of LLMs on longitudinal, multi-turn negotiation data, which is common in real-world political and institutional contexts, and provides a challenging test for tracking context and reasoning over extended interactions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ec0dd436bc7ede2a320921441492b672eb6bb3837aa8e5f78a074f216a2089a5"},"motivation":"LLMs are increasingly being applied to tasks involving institutional and political texts, but existing benchmarks evaluate them on isolated documents or single tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06549","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_streamarena_4967785a","familyId":"bmf_5aa1f09351b6","name":"StreamArena","oneLine":"StreamArena evaluates hour-scale streaming video understanding across 243 full-length videos (avg 88.8 min) with 3,646 open-ended QA pairs, covering real-time perception, historical retrospection, proactive interaction, and multimodal tool use. Includes a standardized runner and LLM-as-judge scorer.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.05703","pdf":"https://arxiv.org/pdf/2608.05703","project":null,"code":"https://github.com/JIA-Lab-research/StreamArena","data":null,"hfPaper":"https://huggingface.co/papers/2608.05703"},"evidence":{"snippet":"We introduce StreamArena, a benchmark for hour-scale, interactive streaming video understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":16,"hfDailySubmittedAt":"2026-08-10T00:00:00.000Z","githubStars":25,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.05703"},"ranking":{"30d":{"score":80,"rank":12,"coverage":0.85,"confidence":"High"},"90d":{"score":70,"rank":76,"coverage":0.7,"confidence":"Medium"}},"description":"StreamArena evaluates hour-scale streaming video understanding across 243 full-length videos (avg 88.8 min) with 3,646 open-ended QA pairs, covering real-time perception, historical retrospection, proactive interaction, and multimodal tool use. Includes a standardized runner and LLM-as-judge scorer.","whyItMatters":"Addresses the lack of benchmarks for long-horizon, interactive streaming video understanding, where short clips and multiple-choice formats allow shortcuts. Provides a rigorous, open-ended evaluation to assess progress in continuous, interactive multimodal agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"671f0a7dc9f6685e9fefd6ba2ee599a227ef4daf4d22af3efd839b23f53bcd68"},"motivation":"Deploying autonomous multimodal agents in continuous, real-world environments requires them to ingest unbounded audio-visual streams and maintain hour-scale memory.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.05703","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"JIA-Lab-research","organizationType":"academic-lab","sourceUrl":"https://github.com/JIA-Lab-research/StreamArena","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_stepjack_2287a7d0","familyId":"bmf_43b09f4a2eb3","name":"StepJack","oneLine":"StepJack evaluates computer-use agents against multi-step indirect prompt injection attacks, where adversarial goals are decomposed into innocuous sub-steps across a chain of pages. It comprises 480 test examples across platforms and instruction types, with attack success rate as the primary metric.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.06477","pdf":"https://arxiv.org/pdf/2608.06477","project":null,"code":"https://github.com/BorealisAI/StepJack","data":null,"hfPaper":"https://huggingface.co/papers/2608.06477"},"evidence":{"snippet":"With this pipeline, we build StepJack, a CUA safety benchmark with 480 test examples.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06477"},"ranking":{"30d":{"score":59,"rank":47,"coverage":0.55,"confidence":"Low"},"90d":{"score":46,"rank":218,"coverage":0.55,"confidence":"Low"}},"description":"StepJack evaluates computer-use agents against multi-step indirect prompt injection attacks, where adversarial goals are decomposed into innocuous sub-steps across a chain of pages. It comprises 480 test examples across platforms and instruction types, with attack success rate as the primary metric.","whyItMatters":"The benchmark addresses the evaluation gap in agent safety against sophisticated, staged prompt injection attacks that previous single-step benchmarks fail to capture, offering a standardized way to assess and compare defenses for computer-use agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"125383de0677db78ed7bbb12a15bc852e83fc2c788fd04f0283a6d6b919c4feb"},"motivation":"Computer-use agents (CUAs) face a growing threat from indirect prompt injection, where adversarial instructions are planted in the environment such as web pages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06477","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"BorealisAI","organizationType":"company-research-lab","sourceUrl":"https://github.com/BorealisAI/StepJack","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_omnimech_c453faaf","familyId":"bmf_67e7a1510b88","name":"OmniMech","oneLine":"OmniMech evaluates vision-language models on four tasks: CAD program synthesis from engineering drawings, diagram-to-3D reasoning, annotation-grounded reasoning, and tool-augmented agentic reasoning using industrial mechanical data with 251k drawings and associated CAD models.","area":"Science & Engineering","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Manufacturing"],"capabilities":["Geometric reasoning"],"topics":["CAD","Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.05539","pdf":"https://arxiv.org/pdf/2608.05539","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.05539"},"evidence":{"snippet":"We introduce OmniMech, the first million-scale benchmark for evaluating VLMs on executable CAD generation from industrial manufacturing data.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.05539"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OmniMech evaluates vision-language models on four tasks: CAD program synthesis from engineering drawings, diagram-to-3D reasoning, annotation-grounded reasoning, and tool-augmented agentic reasoning using industrial mechanical data with 251k drawings and associated CAD models.","whyItMatters":"Existing benchmarks focus on coarse 3D objects; OmniMech addresses the need for evaluating VLMs on fine-grained, dimensioned mechanical designs, providing a standardized testbed to assess progress in executable CAD generation and 3D reconstruction for industrial applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ee1600bbdd2e77b47da49e24a0230a697e8e13c9dc20197380711c11b6c33925"},"motivation":"Recent vision-language models (VLMs) can generate executable CAD programs from images, but existing methods mainly target coarse, general-purpose 3D objects and rarely address the fine-grained geometry and millimeter-level tolerances required in industrial mechanical design.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.05539","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_langchoicebench_33a5b10b","familyId":"bmf_81dd6396539d","name":"LangChoiceBench","oneLine":"LangChoiceBench measures Python preference in project-level code generation across 28 projects and seven software areas, assessing language choice, recommendation-implementation consistency, and language diversity.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Code generation"],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.06041","pdf":"https://arxiv.org/pdf/2608.06041","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.06041"},"evidence":{"snippet":"To bridge this gap, we introduce LangChoiceBench, a project-level code-generation benchmark for measuring Python preference, recommendation-implementation consistency, and language diversity.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06041"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LangChoiceBench measures Python preference in project-level code generation across 28 projects and seven software areas, assessing language choice, recommendation-implementation consistency, and language diversity.","whyItMatters":"Addresses the lack of systematic evaluation of language preference in LLMs, providing a way to compare models on code generation beyond correctness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6ed3bea83cd13cb456ef17e28687d706519bcad6fee3d73d79d6082555c6a14c"},"motivation":"Large language models (LLMs) have been shown to exhibit strong Python preferences when generating project-level code, but there is currently no systematic way to measure this behaviour across new models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06041","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_investlogicbench_44a26030","familyId":"bmf_a9cc3f68edd1","name":"InvestLogicBench","oneLine":"InvestLogicBench evaluates large language models on personalized investment decision-making using 201,247 documented decisions from 151 real-world investors. Each episode traces investor profile, market events, reasoning, decision, and outcome. Tasks include comprehension, profile-conditioned generation, and end-to-end replay.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.06108","pdf":"https://arxiv.org/pdf/2608.06108","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.06108"},"evidence":{"snippet":"We introduce \\textsc{InvestLogicBench}, a process-native benchmark containing 201,247 documented decisions from 151 real-world investors.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06108"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"InvestLogicBench evaluates large language models on personalized investment decision-making using 201,247 documented decisions from 151 real-world investors. Each episode traces investor profile, market events, reasoning, decision, and outcome. Tasks include comprehension, profile-conditioned generation, and end-to-end replay.","whyItMatters":"Existing financial LLM evaluations rely on static QA or terminal profit, which fail to reveal whether actions are profile-consistent or grounded in events. This benchmark targets a gap by assessing process quality and grounding in personalized, consequential settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e16a727c3ab15df9f6562cedff4bbc09cffe4a48865876a5cd79128e01041677"},"motivation":"Investment competence is inherently personalized: the same market evidence can justify different actions for investors with different goals, horizons, portfolios, and risk boundaries.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06108","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_harnessopt-bench_24067463","familyId":"bmf_42c79c547b84","name":"HarnessOpt-Bench","oneLine":"Evaluates LLMs optimizing a target agent's harness (prompts, tools, control flow) under budgeted, stochastic evaluation. Scoring is normalized gain over seed on a held-out test partition, with trusted execution environment enforcing evaluation boundary.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.06301","pdf":"https://arxiv.org/pdf/2608.06301","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.06301"},"evidence":{"snippet":"We introduce HarnessOpt-Bench, a benchmark for end-to-end harness optimization under expensive and stochastic evaluation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":35,"hfDailySubmittedAt":"2026-08-07T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06301"},"ranking":{"30d":{"score":62,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":56,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Evaluates LLMs optimizing a target agent's harness (prompts, tools, control flow) under budgeted, stochastic evaluation. Scoring is normalized gain over seed on a held-out test partition, with trusted execution environment enforcing evaluation boundary.","whyItMatters":"Addresses the gap in measuring automated harness optimization, a discriminative capability needed for improving agentic LLM systems. Provides a protocol for comparing optimizer models under varying harnesses and tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cb38c3cb1edd81673d8340dd632183705f15e21f4f19da9f33d5572fb4765412"},"motivation":"As LLMs are increasingly deployed within agentic systems, their capabilities depend not only on the model weights but also on the harness: the prompts, tools, control flow, memory, and orchestration code surrounding them.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06301","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_gst-bench_e194aeb7","familyId":"bmf_dc90c37b15cc","name":"GST-Bench","oneLine":"GST-Bench is a VQA benchmark for global spatial intelligence in video understanding, covering synthetic videos and human-verified questions. It evaluates VLMs' ability to infer spatial relations from novel viewpoints and map egocentric observations to top-down views.","area":"Vision & 3D","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.05747","pdf":"https://arxiv.org/pdf/2608.05747","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.05747"},"evidence":{"snippet":"To probe the cause of this gap, we construct GST-Bench-Local and find that models, despite strong local spatial understanding under the same task formulation, still fail to consolidate long-horizon observations into a globally consistent scene representation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":46,"hfDailySubmittedAt":"2026-08-07T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.05747"},"ranking":{"30d":{"score":63,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":56,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"GST-Bench is a VQA benchmark for global spatial intelligence in video understanding, covering synthetic videos and human-verified questions. It evaluates VLMs' ability to infer spatial relations from novel viewpoints and map egocentric observations to top-down views.","whyItMatters":"Existing video benchmarks focus on local spatial perception, while GST-Bench targets global spatial awareness over long-horizon videos, addressing a gap in evaluating embodied agents' spatial intelligence. It provides a scoring contract to compare VLMs and highlights a significant performance gap between models and humans.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"67d2fc1f118b7574e621f0d8db9a585f23e343d8f2dc5b26b08bd9c23b8272a4"},"motivation":"Spatial intelligence is fundamental to embodied agents, yet existing benchmarks focus on local spatial perception from single or few viewpoints, overlooking global spatial awareness over continuous, long-horizon visual streams.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.05747","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_gauge_6ce93fe5","familyId":"bmf_a90fd9a9a1e6","name":"GAUGE","oneLine":"GAUGE evaluates physical fidelity of simulation engines and video world models using 22 task families covering rigid bodies, cables, textiles, and deformable objects, with real-world trajectories and calibrated metadata.","area":"Vision & 3D","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.05948","pdf":"https://arxiv.org/pdf/2608.05948","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.05948"},"evidence":{"snippet":"We introduce GAUGE, a real-world-grounded diagnostic benchmark for jointly evaluating how numerical simulators and generative video world models reproduce or deviate from real-world physics.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.05948"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GAUGE evaluates physical fidelity of simulation engines and video world models using 22 task families covering rigid bodies, cables, textiles, and deformable objects, with real-world trajectories and calibrated metadata.","whyItMatters":"Existing evaluations of physical fidelity rely on perceptual similarity or human judgment. GAUGE provides a diagnostic benchmark with grounded physical measurements, enabling systematic comparison of simulators and world models on specific physical principles.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8af04eebd1bfdb552f8a4266debabf9d8e10444d4131d466edf1cbe21c6e08ec"},"motivation":"Physics engines facilitate large-scale training and evaluation for embodied intelligence, while generative video world models are emerging as implicit simulators of future states and interactions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.05948","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_finevo-bench_09755690","familyId":"bmf_46e0c5291976","name":"FinEvo-Bench","oneLine":"FinEvo-Bench evaluates self-evolving agents on 120 real-case-grounded tasks across 20 business scenes in six financial domains, with institution-provided procedures and rubrics for quality and compliance.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["Self-Evolution","Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.06144","pdf":"https://arxiv.org/pdf/2608.06144","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.06144"},"evidence":{"snippet":"We introduce FinEvo-Bench, a longitudinal benchmark with 120 real-case-grounded tasks, 20 business scenes across six financial domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06144"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FinEvo-Bench evaluates self-evolving agents on 120 real-case-grounded tasks across 20 business scenes in six financial domains, with institution-provided procedures and rubrics for quality and compliance.","whyItMatters":"Most agent benchmarks treat tasks independently and cannot measure learning from experience. FinEvo-Bench provides a longitudinal evaluation that measures both professional performance and self-evolution ability, filling a gap in agent benchmarking.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a4b4a94906f9eef5f72c45bd3104639dee614322b0ec00ee21e698d1a919a589"},"motivation":"Most agent benchmarks evaluate tasks independently and cannot measure whether experience from one task helps with later tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06144","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_epibench_00d62089","familyId":"bmf_12b5a09e9b7d","name":"EpiBench","oneLine":"EpiBench evaluates LLMs' epitope reasoning from antibody and antigen sequences across five tasks: targetable region discovery, epitope identification, binning, functional assessment, and escape assessment.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.06022","pdf":"https://arxiv.org/pdf/2608.06022","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.06022"},"evidence":{"snippet":"To address this gap, we introduce EpiBench, a closed-book, sequence-based, and automatically scorable benchmark for evaluating epitope reasoning in LLMs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06022"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EpiBench evaluates LLMs' epitope reasoning from antibody and antigen sequences across five tasks: targetable region discovery, epitope identification, binning, functional assessment, and escape assessment.","whyItMatters":"Previous epitope resources focus on isolated prediction tasks and do not evaluate epitope-centered decisions across the antibody development workflow. EpiBench provides a closed-book, automatically scorable benchmark for this purpose.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b85c65ee3ef651e110795f44fe1baa689b0c011473ea549ca08f2448835915c3"},"motivation":"Epitopes determine where antibodies bind antigens and shape downstream therapeutic properties such as functional blockade and escape resistance, making epitope understanding central to antibody drug discovery.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06022","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_ecoagent-bench_99f855a3","familyId":"bmf_5ccd0648d9b6","name":"EcoAgent-Bench","oneLine":"EcoAgent-Bench evaluates LLM agents' economic decision-making under budget constraints across 304 tasks in five families, with priced actions and metrics for micro accuracy and economic consistency.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.05519","pdf":"https://arxiv.org/pdf/2608.05519","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.05519"},"evidence":{"snippet":"We introduce EcoAgent-Bench, in which every task specifies priced actions and an explicit budget.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.05519"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EcoAgent-Bench evaluates LLM agents' economic decision-making under budget constraints across 304 tasks in five families, with priced actions and metrics for micro accuracy and economic consistency.","whyItMatters":"Existing agent benchmarks measure task completion without considering cost-effectiveness. EcoAgent-Bench explicitly tests the trade-off between completion and resource use, providing a distinct evaluation dimension.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c3fe677923cbaccb685dda4d50f22346ef6e985d82cec5774787a6cd75fca190"},"motivation":"Agent benchmarks usually measure task completion and treat resource use as an auxiliary statistic.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.05519","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_c-suitebench_9e6fa218","familyId":"bmf_77b58031dbe0","name":"C-SUITEBENCH","oneLine":"C-SUITEBENCH evaluates multimodal LLMs as CEOs on five decision tasks under paired text-only and multimodal conditions across 50 scenarios, focusing on evidence-centric reasoning and constraint satisfaction.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.05864","pdf":"https://arxiv.org/pdf/2608.05864","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.05864"},"evidence":{"snippet":"We introduce C-SUITEBENCH, a controlled multimodal benchmark that includes five decision tasks under paired text-only and multimodal conditions across 50 scenarios.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.05864"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"C-SUITEBENCH evaluates multimodal LLMs as CEOs on five decision tasks under paired text-only and multimodal conditions across 50 scenarios, focusing on evidence-centric reasoning and constraint satisfaction.","whyItMatters":"Existing executive decision benchmarks are text-only, leaving unclear whether models can integrate visual evidence. C-SUITEBENCH reveals a multimodal integration paradox where visual inputs can degrade constrained allocation, informing selective grounding strategies.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"042d1bc558a788a5b6c088e56b9e4d4379e49eda315bc451067f00da7927e8d9"},"motivation":"Large language models are increasingly applied as autonomous decision-making agents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.05864","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_skill-use_dfe395c1","familyId":"bmf_43888fe52bed","name":"Skill-Use","oneLine":"Skill-Use evaluates skill use in agentic harnesses through 79 real skills and 177 executable tasks across nine domains, measuring trigger, compliance, and boundary adherence with a combined SU score.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04828","pdf":"https://arxiv.org/pdf/2608.04828","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.04828"},"evidence":{"snippet":"We introduce Skill-Use, a benchmark that evaluates skill use under progressive disclosure, where an agent sees only a skill's name and short description and must retrieve the full procedure before following it.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04828"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Skill-Use evaluates skill use in agentic harnesses through 79 real skills and 177 executable tasks across nine domains, measuring trigger, compliance, and boundary adherence with a combined SU score.","whyItMatters":"Agent evaluations often focus on task success, not whether agents can autonomously identify and apply relevant skills. Skill-Use isolates skill retrieval and usage under progressive disclosure, showing harness-dependent capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6a7c1ac3397cd3107e6e682fc0e118b50f3b5b22a71cde757279c5a932636cb0"},"motivation":"Large language model (LLM) agents increasingly rely on skills, structured documents that specify when to act, which procedure to follow, and which tools are allowed.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04828","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_omnirouting_ca769ccd","familyId":"bmf_f17dee6f17fd","name":"OmniRouting","oneLine":"OmniRouting evaluates LLMs on PCB routing reasoning under real-world constraints with 1,681 industrial designs across four tasks: geometric routing, design-rule-aware routing, electrical functionality, and tool-augmented agentic routing.","area":"Multimodal","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Semiconductors"],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04434","pdf":"https://arxiv.org/pdf/2608.04434","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.04434"},"evidence":{"snippet":"To bridge this gap, we introduce OmniRouting, the first large-scale benchmark designed to evaluate LLMs on printed-circuit-board (PCB) routing reasoning under real-world industrial design-rule, manufacturability, and connectivity constraints.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04434"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OmniRouting evaluates LLMs on PCB routing reasoning under real-world constraints with 1,681 industrial designs across four tasks: geometric routing, design-rule-aware routing, electrical functionality, and tool-augmented agentic routing.","whyItMatters":"Current benchmarks do not cover routing under strict geometric, topological, and electrical constraints. OmniRouting fills this gap, exposing limitations in path planning and rule adherence for LMMs in EDA.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a2fef8a361f3635d8a96ce754e82771c093093fe7dea6a0095571c299168c807"},"motivation":"Recent large language models (LLMs) have demonstrated remarkable progress in constraint-aware navigation, maze reasoning, and graph reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04434","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_omniedit-bench_6a691145","familyId":"bmf_48dbdad242c2","name":"OmniEdit-Bench","oneLine":"OmniEdit-Bench evaluates instruction-based video editing with dimensions including spatial, temporal, audio, and reference-based editing, and assesses accuracy, preservation, realism, and consistency with an accuracy-aware penalty.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.05049","pdf":"https://arxiv.org/pdf/2608.05049","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.05049"},"evidence":{"snippet":"To address these issues, we introduce a comprehensive and structured benchmark for IVE.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.05049"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OmniEdit-Bench evaluates instruction-based video editing with dimensions including spatial, temporal, audio, and reference-based editing, and assesses accuracy, preservation, realism, and consistency with an accuracy-aware penalty.","whyItMatters":"Existing video editing benchmarks have limited task coverage and metrics that fail to measure instruction fidelity. OmniEdit-Bench provides a structured evaluation that prevents incorrect edits from receiving inflated scores.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"83b795e2383ec191214c586db1be83d6d36a86f7c73504bd0cfe4dc63ea28595"},"motivation":"Instruction-based video editing (IVE) is an emerging field with broad applications, yet evaluating editing models remains challenging.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.05049","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_nolli_b5bb5859","familyId":"bmf_9e92e28bd0ca","name":"NOLLI","oneLine":"NOLLI is a procedurally generated English-Korean puzzle benchmark with 15 puzzle types (25 tasks, 7,500 items). Each instance is seed-regenerable, verified to have a unique solution, and scored deterministically. Difficulty is calibrated behaviorally to target accuracy bands.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04397","pdf":"https://arxiv.org/pdf/2608.04397","project":null,"code":"https://github.com/HAE-RAE/NOLLI","data":null,"hfPaper":"https://huggingface.co/papers/2608.04397"},"evidence":{"snippet":"We introduce NOLLI, a procedurally generated English-Korean puzzle benchmark designed to diagnose where Korean performance gaps arise.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":23,"hfDailySubmittedAt":"2026-08-06T00:00:00.000Z","githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04397"},"ranking":{"30d":{"score":72,"rank":26,"coverage":0.85,"confidence":"High"},"90d":{"score":57,"rank":157,"coverage":0.7,"confidence":"Medium"}},"description":"NOLLI is a procedurally generated English-Korean puzzle benchmark with 15 puzzle types (25 tasks, 7,500 items). Each instance is seed-regenerable, verified to have a unique solution, and scored deterministically. Difficulty is calibrated behaviorally to target accuracy bands.","whyItMatters":"NOLLI addresses the lack of controlled cross-lingual benchmarks that separate presentation language from reasoning difficulty, enabling diagnosis of where model performance gaps arise across languages and writing systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d5b263a38d9fcdd605bc1a4663b7748e42e1fd043a37ffb89972162a092a1fd3"},"motivation":"We introduce NOLLI, a procedurally generated English-Korean puzzle benchmark designed to diagnose where Korean performance gaps arise.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04397","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HAE-RAE","organizationType":"academic-lab","sourceUrl":"https://github.com/HAE-RAE/NOLLI","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_lunar_024583f6","familyId":"bmf_9738b6bf3ae3","name":"LUNAR","oneLine":"LUNAR is a benchmark for evaluating LLM personalization from longitudinal app interaction logs across domains like clothing, food, housing, and mobility. It uses a synthetic data pipeline and evaluates 19 LLMs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.05246","pdf":"https://arxiv.org/pdf/2608.05246","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.05246"},"evidence":{"snippet":"To address this gap, we introduce LUNAR, the first benchmark for evaluating how LLMs personalize responses from longitudinal app interaction histories across universal daily-life domains, including clothing, food, housing, and mobility.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.05246"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LUNAR is a benchmark for evaluating LLM personalization from longitudinal app interaction logs across domains like clothing, food, housing, and mobility. It uses a synthetic data pipeline and evaluates 19 LLMs.","whyItMatters":"LUNAR could address the need for evaluating cross-domain personalization from behavioral logs, but without public artifacts or clear scoring details, its utility is unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d4c52954918577de25a9629f41e917e2db0ebeadc7a853b7d3953571e3f18ec8"},"motivation":"Existing personalized LLM benchmarks primarily rely on textual personas or isolated behavioral signals, providing limited evaluation of cross-domain behavioral personalization, where responses must be grounded in heterogeneous daily-life activities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.05246","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_finreportbench_5d3a5df7","familyId":"bmf_33ecc6c20ee7","name":"FinReportBench","oneLine":"FinReportBench evaluates institution-grade financial report generation using 244 bilingual tasks sourced from 10,000 financial research records. It uses a 35-item rubric covering deliverability, report identity, and institutional completeness, with three judge families.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04374","pdf":"https://arxiv.org/pdf/2608.04374","project":null,"code":"https://github.com/MisterBrookT/finreportbench","data":null,"hfPaper":"https://huggingface.co/papers/2608.04374"},"evidence":{"snippet":"We introduce FinReportBench, an expert-grounded benchmark for measuring and improving institution-grade financial report generation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04374"},"ranking":{"30d":{"score":56,"rank":52,"coverage":0.55,"confidence":"Low"},"90d":{"score":44,"rank":242,"coverage":0.55,"confidence":"Low"}},"description":"FinReportBench evaluates institution-grade financial report generation using 244 bilingual tasks sourced from 10,000 financial research records. It uses a 35-item rubric covering deliverability, report identity, and institutional completeness, with three judge families.","whyItMatters":"It fills the gap in evaluating long-form financial reports for institutional delivery, providing a reliable rubric and public artifacts to measure and improve report generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1dd2d50bc647edddfc6fe7ca60b7b3687f351d68dca43bbe2d02a002bd9c4d03"},"motivation":"Large language models can produce fluent financial analysis, but fluency alone does not establish whether a report is suitable for institutional delivery.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04374","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_euroexec_53125f62","familyId":"bmf_2d0194cba00a","name":"EuroExec","oneLine":"EuroExec is an expert-authored benchmark of 413 open-ended European executive tasks evaluated by human experts. It uses a multi-attribute rubric and preference ranking to compute a Solve Rate.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.04549","pdf":"https://arxiv.org/pdf/2608.04549","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.04549"},"evidence":{"snippet":"We dedicate more than 4,000 human expert hours to evaluate a selection of six frontier LLMs on a member of this class of problems: EuroExec, our introduced human expert-based benchmark composed of 413 open-ended long-form European executive tasks authored by 47 vetted domain experts, each question drawn from experience in a real case.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04549"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EuroExec is an expert-authored benchmark of 413 open-ended European executive tasks evaluated by human experts. It uses a multi-attribute rubric and preference ranking to compute a Solve Rate.","whyItMatters":"It addresses evaluation of open-ended, complex tasks where subjective judgment is key, but the lack of public artifacts makes its reuse uncertain.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b52fc3bf1603b151ec3e788412ca59c9da373be792b76784774be06a631a88a9"},"motivation":"Frontier LLMs are increasingly put to use on open-ended complex questions, different in nature from the ones they are typically evaluated on.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04549","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_egoafford_be5c2b17","familyId":"bmf_7252a0a76b4f","name":"EgoAfford","oneLine":"EgoAfford is a benchmark for egocentric referring segmentation with task-oriented affordance grounding, comprising 15.5k images and 102 real images. It includes EgoLens, a 3B MLLM reference model.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04533","pdf":"https://arxiv.org/pdf/2608.04533","project":"https://egoafford.github.io","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.04533"},"evidence":{"snippet":"We introduce EgoAfford, a benchmark designed to connect these three aspects.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04533"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EgoAfford is a benchmark for egocentric referring segmentation with task-oriented affordance grounding, comprising 15.5k images and 102 real images. It includes EgoLens, a 3B MLLM reference model.","whyItMatters":"It addresses the need for connecting perception and planning in tabletop tasks, but the lack of public artifacts and scoring details limits its immediate utility.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d83b9224f5f13677b55e6ee0c150748cb38e0571156e38b0723803cd61d93ea4"},"motivation":"Part-level affordance grounding has advanced the localization of functional object regions associated with elemental actions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04533","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_contextweave_c388d0b9","familyId":"bmf_a571d6806b2e","name":"ContextWeave","oneLine":"ContextWeave is a longitudinal benchmark for evaluating memory in office workflows, with 1,005 executable tasks from 14 participants. It measures workspace quality and preference alignment.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04830","pdf":"https://arxiv.org/pdf/2608.04830","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.04830"},"evidence":{"snippet":"We introduce ContextWeave, a longitudinal benchmark that evaluates whether recalled experience improves downstream agent performance in realistic office-work streams.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04830"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ContextWeave is a longitudinal benchmark for evaluating memory in office workflows, with 1,005 executable tasks from 14 participants. It measures workspace quality and preference alignment.","whyItMatters":"It addresses evaluation of memory in long-horizon agent workflows, but the lack of public artifacts and scoring details makes it non-reusable without further information.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"696b147cae0380d7a7acd8296bd7539985e234930bdd8b98236ea664f191fc21"},"motivation":"Memory is essential as language agents move from isolated tasks to long-horizon, stateful workflows, yet existing evaluations often reduce it to retrieval or question answering.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04830","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_complexityworld_8718400b","familyId":"bmf_a96f895480e9","name":"ComplexityWorld","oneLine":"ComplexityWorld is a benchmark of 390 visual decision-making tasks across 39 worlds, scored by an executable verifier. It evaluates VLMs on tasks requiring global constraint satisfaction.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07584","pdf":"https://arxiv.org/pdf/2608.07584","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07584"},"evidence":{"snippet":"We introduce COMPLEXITYWORLD, a benchmark of 390 tasks across 39 domain-inspired visual worlds and 29 decision categories.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07584"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ComplexityWorld is a benchmark of 390 visual decision-making tasks across 39 worlds, scored by an executable verifier. It evaluates VLMs on tasks requiring global constraint satisfaction.","whyItMatters":"It targets a persistent visual-to-decision bottleneck in VLMs, but the lack of public artifacts and scoring details hinders independent verification.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"14061cd27b8e0b24c6793af467a056e91f670972e423bb458b5a211a38acd4aa"},"motivation":"Vision-language models (VLMs) have made rapid progress in visual perception and increasingly support real-world tasks that depend on images.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07584","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_clip-cc-bench_6e2be6c1","familyId":"bmf_250f7a769d65","name":"CLIP-CC-Bench","oneLine":"CLIP-CC-Bench evaluates paragraph-level video description quality using 5-hour movie content and expert-written references. It scores 17 VLMs via coarse- and fine-grained semantic matching with five LLM-based embedding judges.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.04302","pdf":"https://arxiv.org/pdf/2608.04302","project":null,"code":"https://github.com/Multimodal-Intelligence-Lab/CLIP-CC-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.04302"},"evidence":{"snippet":"We introduce CLIP-CC-Bench, an evaluation suite for long-form video description built from 5 hours of movie content segmented into 90-second clips, each paired with an expert-written paragraph-style reference.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":"2026-08-10T00:00:00.000Z","githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04302"},"ranking":{"30d":{"score":49,"rank":79,"coverage":0.85,"confidence":"High"},"90d":{"score":40,"rank":266,"coverage":0.7,"confidence":"Medium"}},"description":"CLIP-CC-Bench evaluates paragraph-level video description quality using 5-hour movie content and expert-written references. It scores 17 VLMs via coarse- and fine-grained semantic matching with five LLM-based embedding judges.","whyItMatters":"It fills the gap in long-form video description benchmarks, providing a reliable framework with public scripts and data for reproducible evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"eab2cf54c1d7768789183433e46ea8b124eb03232526c372d3be4133bcf43833"},"motivation":"Benchmarking video-language models has largely focused on short clips and single-sentence metrics, leaving open whether current systems can generate accurate long-form, paragraph-level descriptions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"and presented at EvalMG 2026, the Second Workshop on Evaluation for Multimodal Generation, co-located with ACM SIGIR 202","evidence":"Accepted and presented at EvalMG 2026, the Second Workshop on Evaluation for Multimodal Generation, co-located with ACM SIGIR 2026","evidenceUrl":"https://arxiv.org/abs/2608.04302","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"and presented at EvalMG 2026, the Second Workshop on Evaluation for Multimodal Generation, co-located with ACM SIGIR 202","reviewStatus":"accepted","decisionRaw":"Accepted and presented at EvalMG 2026, the Second Workshop on Evaluation for Multimodal Generation, co-located with ACM SIGIR 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.04302","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted and presented at EvalMG 2026, the Second Workshop on Evaluation for Multimodal Generation, co-located with ACM SIGIR 2026","level":"author-claim"}]}],"publishers":[{"name":"Multimodal Intelligence Lab","organizationType":"academic-lab","sourceUrl":"https://github.com/Multimodal-Intelligence-Lab/CLIP-CC-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_active-swe_8c4ac0e6","familyId":"bmf_2cbe3d83caa7","name":"Active-SWE","oneLine":"Active-SWE evaluates coding agents on proactive bug fixing: detecting and fixing multiple bugs without issue reports. It includes 1,663 tasks across six bug categories and eight languages, with stages for recorded bugs, potential bugs, and judge validation.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04682","pdf":"https://arxiv.org/pdf/2608.04682","project":null,"code":"https://github.com/XLearning-SCU/Active-SWE","data":null,"hfPaper":"https://huggingface.co/papers/2608.04682"},"evidence":{"snippet":"To address this, we introduce Active-SWE, a benchmark for evaluating coding agents on proactively discovering and fixing multiple bugs without report guidance, covering 1,663 tasks across six bug categories and eight languages.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":4,"hfDailySubmittedAt":null,"githubStars":51,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04682"},"ranking":{"30d":{"score":75,"rank":19,"coverage":0.85,"confidence":"High"},"90d":{"score":73,"rank":58,"coverage":0.7,"confidence":"Medium"}},"description":"Active-SWE evaluates coding agents on proactive bug fixing: detecting and fixing multiple bugs without issue reports. It includes 1,663 tasks across six bug categories and eight languages, with stages for recorded bugs, potential bugs, and judge validation.","whyItMatters":"Existing SWE benchmarks assume detailed issue reports are available, which is unrealistic. Active-SWE fills the gap by testing agents' ability to discover and fix bugs proactively, a capability that current state-of-the-art agents struggle with, providing a more realistic evaluation of coding agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a7408611b0f4eb01effd4883f84e64de9af4d4454e81497eff0d21095bb55c7b"},"motivation":"Coding agents powered by large language models (LLMs) are increasingly adopted in software engineering (SWE) scenarios, capable of fixing a specific bug in large-scale codebase.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04682","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"XLearning-SCU","organizationType":"academic-lab","sourceUrl":"https://github.com/XLearning-SCU/Active-SWE","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_weclawarena_f743496b","familyId":"bmf_4cf9742a8ee1","name":"WeClawArena","oneLine":"WeClawArena is a benchmark and runtime sandbox for multi-party owned-agent collaboration over personal workspaces. It contains 124 base tasks across six domains, expanded into 620 scenario variants with benign and attack-vector conditions, and reports task success and attack success separately.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03499","pdf":"https://arxiv.org/pdf/2608.03499","project":null,"code":"https://github.com/kingofspace0wzz/WeClawArena","data":null,"hfPaper":"https://huggingface.co/papers/2608.03499"},"evidence":{"snippet":"We introduce WeClawArena, an auditable benchmark and runtime sandbox for multi-party owned-agent collaboration over personal workspaces.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":"2026-08-11T00:00:00.000Z","githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03499"},"ranking":{"30d":{"score":53,"rank":59,"coverage":0.85,"confidence":"High"},"90d":{"score":42,"rank":260,"coverage":0.7,"confidence":"Medium"}},"description":"WeClawArena is a benchmark and runtime sandbox for multi-party owned-agent collaboration over personal workspaces. It contains 124 base tasks across six domains, expanded into 620 scenario variants with benign and attack-vector conditions, and reports task success and attack success separately.","whyItMatters":"Existing agent benchmarks do not provide an end-to-end sandbox for verifiable cross-user agent collaboration with realistic digital workspaces. WeClawArena enables evaluation of both collaborative task utility and security risks in human-centered agent networks, supporting diagnosis of privacy leakage, poisoned evidence, and invalid authority paths.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2a0bad212dc0d7814ba1d3019a723e8879bca65b589da7017ae5b891d1371589"},"motivation":"Recent advances in persistent personal-agent frameworks are making human-centered agent networks realistic deployment targets: each user can be served by an AI agent that acts on the user's behalf, maintains state, and communicates with other agents through social and task relations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03499","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"kingofspace0wzz","organizationType":"community","sourceUrl":"https://github.com/kingofspace0wzz/WeClawArena","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_vibe_30ded12f","familyId":"bmf_8368ec5f1a15","name":"VIBE","oneLine":"VIBE is a benchmark for entity-centered affective profiling of LLM outputs in Valence-Arousal-Dominance space, introducing a measurement contract with scalar favorability, response-level and target-directed VAD, and an Affective Passport reporting format.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03810","pdf":"https://arxiv.org/pdf/2608.03810","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03810"},"evidence":{"snippet":"We introduce VIBE, a benchmark for entity-centered affective profiling of LLM outputs in Valence-Arousal-Dominance (VAD) space.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03810"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"VIBE is a benchmark for entity-centered affective profiling of LLM outputs in Valence-Arousal-Dominance space, introducing a measurement contract with scalar favorability, response-level and target-directed VAD, and an Affective Passport reporting format.","whyItMatters":"Existing sentiment and emotion benchmarks do not combine target-directed VAD attribution with an explicit scorer contract and passport reporting. VIBE provides a standardized way to report affective profiles, supporting practice-oriented evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ad711d62b543408e9ddd88bcf72ef384f476ef4e8c6b77ce19461653869a0fd4"},"motivation":"Large language models routinely describe socially salient targets, including political figures, countries, religions, organizations, historical events, and social groups, encoding affective framing alongside factual content: a target may appear favorable or threatening, calm or conflictual, powerful or vulnerable.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03810","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_societybench_8a44f535","familyId":"bmf_2d0da2d0dcde","name":"SocietyBench","oneLine":"SocietyBench is a benchmark for forecasting counterfactual social-world evolution, collecting web news and social-media posts to build timelines and generating forecasting questions scored on probability calibration and temporal accuracy.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04009","pdf":"https://arxiv.org/pdf/2608.04009","project":"https://co-minder.github.io/Societybench","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.04009"},"evidence":{"snippet":"We introduce SocietyBench, an end-to-end benchmark that takes a one-line event topic, collects Web news and social-media posts across five platforms, distills them into a date-indexed timeline that keeps factual events and a public-opinion layer separate, and then turns every cutoff date on that timeline into an audited bank of forecasting questions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04009"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SocietyBench is a benchmark for forecasting counterfactual social-world evolution, collecting web news and social-media posts to build timelines and generating forecasting questions scored on probability calibration and temporal accuracy.","whyItMatters":"Current benchmarks focus on task completion, not on social understanding and forecasting. SocietyBench measures how well models understand and predict social events, providing a complementary evaluation axis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c56bd9b46af2a4422d3b738fef97316b2f035d9027b08c18675b5364c001d03e"},"motivation":"Large language models (LLMs), and the agents built on top of them, are now benchmarked heavily on whether they can finish a task -- fix a bug, drive a browser, operate a GUI.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04009","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_signpost-bench_c7de79e5","familyId":"bmf_dec2c32e4dca","name":"SIGNPOST-Bench","oneLine":"SIGNPOST-Bench evaluates text-vision conflict resolution in multimodal large language models via a counterfactual benchmark of image variants (Original, Blank, Similar, Random, Adversarial) for visual geolocation. It includes 5,111 groups and 25,555 variants, with metrics for localization error and target-directed shifts.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04244","pdf":"https://arxiv.org/pdf/2608.04244","project":null,"code":"https://github.com/inorganicwriter/SIGNPOST-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.04244"},"evidence":{"snippet":"We introduce SIGNPOST-Bench, a controlled counterfactual benchmark for evaluating text-vision conflict resolution.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":"2026-08-06T00:00:00.000Z","githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04244"},"ranking":{"30d":{"score":57,"rank":49,"coverage":0.85,"confidence":"High"},"90d":{"score":45,"rank":231,"coverage":0.7,"confidence":"Medium"}},"description":"SIGNPOST-Bench evaluates text-vision conflict resolution in multimodal large language models via a counterfactual benchmark of image variants (Original, Blank, Similar, Random, Adversarial) for visual geolocation. It includes 5,111 groups and 25,555 variants, with metrics for localization error and target-directed shifts.","whyItMatters":"Existing benchmarks rarely reveal how MLLMs arbitrate conflicting text and visual evidence. SIGNPOST-Bench provides a controlled framework to measure robustness to conflicting scene text, showing that localization performance degrades substantially under adversarial text edits.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8d133c017d3c6059a903e463c0466459c1859cd3a889e9bfb9adfce05d290ab5"},"motivation":"Multimodal large language models (MLLMs) make grounded predictions in real-world scenes by combining visual and textual cues, yet existing benchmarks rarely reveal how they arbitrate between these evidence sources when they conflict.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04244","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"inorganicwriter","organizationType":"community","sourceUrl":"https://github.com/inorganicwriter/SIGNPOST-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_segabench_e418e029","familyId":"bmf_7c6281aa1c49","name":"SeGaBench","oneLine":"SeGaBench is an executable benchmark containing 120 cases (100 synthetic, 20 source-backed) to test whether LLMs can recover semantic optimization opportunities that compilers miss, with hidden enabling semantics, oracle artifacts, and validators.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.PL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.03983","pdf":"https://arxiv.org/pdf/2608.03983","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03983"},"evidence":{"snippet":"We introduce SeGaBench, an executable benchmark containing 100 synthetic and 20 source-backed cases spanning low-level assumptions, data-structure invariants, and high-level semantic lifting.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03983"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SeGaBench is an executable benchmark containing 120 cases (100 synthetic, 20 source-backed) to test whether LLMs can recover semantic optimization opportunities that compilers miss, with hidden enabling semantics, oracle artifacts, and validators.","whyItMatters":"Compilers miss profitable transformations when enabling semantics are absent from the program representation. SeGaBench evaluates whether LLMs can recover such semantics and produce validated, performance-improving artifacts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d2b994507120a294e65116dbc46b3ccb8ae5d84c4912a6cf3e33f516c33fe3df"},"motivation":"Optimizing compilers miss profitable transformations when their enabling semantics are absent from the analyzed program representation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03983","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_past-bench_b04dad58","familyId":"bmf_d6508586cac7","name":"PAST-Bench","oneLine":"PAST-Bench evaluates recursive self-improvement in personal AI agents by testing whether retained experience improves performance on future tasks. It spans 26 scenarios and 204 episodes across memory, procedural reuse, information gathering, and update capabilities, with matched persistence on/off controls.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Self-Evolution","Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04003","pdf":"https://arxiv.org/pdf/2608.04003","project":null,"code":"https://github.com/Gen-Verse/PAST-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.04003"},"evidence":{"snippet":"We introduce PAST-Bench, a benchmark designed to isolate this question.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":33,"hfDailySubmittedAt":"2026-08-05T00:00:00.000Z","githubStars":22,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04003"},"ranking":{"30d":{"score":83,"rank":9,"coverage":0.85,"confidence":"High"},"90d":{"score":70,"rank":77,"coverage":0.7,"confidence":"Medium"}},"description":"PAST-Bench evaluates recursive self-improvement in personal AI agents by testing whether retained experience improves performance on future tasks. It spans 26 scenarios and 204 episodes across memory, procedural reuse, information gathering, and update capabilities, with matched persistence on/off controls.","whyItMatters":"Whether personal agents actually improve from retained experience has not been systematically tested. PAST-Bench provides a controlled benchmark to measure and attribute cross-session improvement, distinguishing capability gains from pathway evidence.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0ef8aa96463f20467ad56ab00c20214e33f953a38a56dd9a61d0b003234a832f"},"motivation":"Recursive self-improvement requires agents to turn accumulated experience into better future behavior.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04003","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Gen-Verse","organizationType":"community","sourceUrl":"https://github.com/Gen-Verse/PAST-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_parambench_25e30479","familyId":"bmf_00f78109a13a","name":"ParamBench","oneLine":"ParamBench is a benchmark for evaluating LLM tool call parameter generation, built from real cloud-network APIs with difficulty tiers and exact match metrics. It is used to evaluate the proposed probe-guided training framework.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03071","pdf":"https://arxiv.org/pdf/2608.03071","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03071"},"evidence":{"snippet":"To support systematic evaluation, we release ParamBench, a benchmark built from real cloud-network APIs that categorizes every instance into five difficulty levels according to parameter nesting depth, cross-parameter dependencies, and the reasoning required to derive values from earlier calls.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03071"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ParamBench is a benchmark for evaluating LLM tool call parameter generation, built from real cloud-network APIs with difficulty tiers and exact match metrics. It is used to evaluate the proposed probe-guided training framework.","whyItMatters":"Tool call parameter correctness is critical for execution yet understudied. ParamBench provides a systematic evaluation of parameter generation across difficulty levels, showing large improvements from probe-guided methods.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"86610886987ac92dc1bf815d5db268a7fa427bc436c85d171ce3f15d019d96de"},"motivation":"Large language model agents derive much of their capability from tool use.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03071","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_multiglobeqa_73949294","familyId":"bmf_6185f11d15a8","name":"MultiGlobeQA","oneLine":"MultiGlobeQA evaluates geospatial reasoning in large language models across 46,060 question-answer pairs in 17 languages, covering 14 spatial-function families and 15 answer formats, with ground truth from three knowledge graphs.","area":"Language & Knowledge","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Logistics"],"capabilities":["Reasoning","Factuality"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03882","pdf":"https://arxiv.org/pdf/2608.03882","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03882"},"evidence":{"snippet":"We introduce MultiGlobeQA, a multilingual benchmark of 46,060 question-answer pairs spanning 14 spatial-function families and 15 answer formats, with execution-based ground truth over three knowledge graphs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03882"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MultiGlobeQA evaluates geospatial reasoning in large language models across 46,060 question-answer pairs in 17 languages, covering 14 spatial-function families and 15 answer formats, with ground truth from three knowledge graphs.","whyItMatters":"Current geospatial benchmarks are limited in language coverage and geographic control. This benchmark aims to provide a broader, multilingual evaluation to identify specific failures in computational reasoning over geographic knowledge.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c80cca706fd7ab8e85516160919e62f76cce7721f812ffcf4b00855bed45ae84"},"motivation":"Geospatial reasoning, i.e., computing distances, containment, and other spatial relations over real-world entities, is central to navigation and logistics, yet large language models (LLMs) struggle with the required geometric and topological computation despite storing considerable geographic knowledge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03882","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_mt-web2code_5f98b4bc","familyId":"bmf_19c14f7a2f67","name":"MT-Web2Code","oneLine":"MT-Web2Code evaluates coding agents on multi-turn web UI reconstruction and modification tasks across 102 tasks in 16 domains, with a dual-axis protocol measuring target-region fidelity and preservation of unaffected content.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03474","pdf":"https://arxiv.org/pdf/2608.03474","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03474"},"evidence":{"snippet":"To bridge this gap, we introduce MT-Web2Code, the first multimodal coding benchmark for multi-turn Macro-Level Regional Reconstruction and Micro-Level Localized Modification, which contains 102 tasks spanning 16 vertical domains.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03474"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MT-Web2Code evaluates coding agents on multi-turn web UI reconstruction and modification tasks across 102 tasks in 16 domains, with a dual-axis protocol measuring target-region fidelity and preservation of unaffected content.","whyItMatters":"Existing benchmarks focus on single-turn full-page generation, missing the iterative workflow of real frontend engineering. This benchmark aims to fill that gap and identify weaknesses in multi-turn UI coding agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e1f5c6f94f724e97ea277824df0b568dc710f1b4baedf7f7cc40b530bca4a3a7"},"motivation":"Recent advances in Large Vision-Language Models (LVLMs) have demonstrated impressive capabilities in web UI generation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03474","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ms-mlb_c7714a98","familyId":"bmf_23c6680cf29b","name":"MS-MLB","oneLine":"MS-MLB evaluates machine learning models for classifying multiple sclerosis versus healthy controls from whole blood RNA expression data (GSE17048). It uses a shared pipeline with nested cross-validation and a holdout set, and reports the MS Research Score composite metric.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.05196","pdf":"https://arxiv.org/pdf/2608.05196","project":null,"code":"https://github.com/duckyquang/MS-MLB","data":null,"hfPaper":"https://huggingface.co/papers/2608.05196"},"evidence":{"snippet":"This paper presents MS-MLB (Multiple Sclerosis Machine Learning Benchmark), a reproducible open benchmark for machine learning based MS research classification from whole blood RNA expression data.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.05196"},"ranking":{"30d":{"score":34,"rank":133,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":356,"coverage":0.55,"confidence":"Low"}},"description":"MS-MLB evaluates machine learning models for classifying multiple sclerosis versus healthy controls from whole blood RNA expression data (GSE17048). It uses a shared pipeline with nested cross-validation and a holdout set, and reports the MS Research Score composite metric.","whyItMatters":"Existing MS transcriptomic studies lack reproducible and standardized evaluation. This benchmark provides a leakage-controlled pipeline and an external model submission pathway for comparable research comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e481c1e43b4e5bd038667b3a3a00409fa49fd6f3600f95c41c72fc7d37c08531"},"motivation":"Multiple sclerosis (MS) is diagnosed through clinical assessment, magnetic resonance imaging, laboratory evidence when appropriate, and exclusion of better explanations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.05196","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Synthica Research Group","organizationType":"community","sourceUrl":"https://github.com/duckyquang/MS-MLB","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_medpic-bench_f4184b8f","familyId":"bmf_f9677fb45e23","name":"MedPIC-Bench","oneLine":"MedPIC-Bench evaluates patient-specific medication-safety reasoning in LLMs using guideline-following and paired counterfactual questions, with 467 questions annotated across six dimensions.","area":"Safety & Trustworthiness","applicationDomains":["Cybersecurity","Health & Life Sciences"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity","Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Safety","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.03028","pdf":"https://arxiv.org/pdf/2608.03028","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03028"},"evidence":{"snippet":"To address this gap, we introduce MedPIC-Bench, a benchmark of source-verifiable recommendations and expert-validated questions for patient-specific medication-safety reasoning.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03028"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MedPIC-Bench evaluates patient-specific medication-safety reasoning in LLMs using guideline-following and paired counterfactual questions, with 467 questions annotated across six dimensions.","whyItMatters":"Static medication-safety accuracy may not reflect whether models use patient information to determine rule applicability. This benchmark aims to make conditional rule application measurable.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"20a8f713b6ed5f6b35987cf9f11377968ea22af6f25c06e64ea06d1ae59835cb"},"motivation":"Applying a valid medication-safety rule when its patient-specific conditions are not met can produce an incorrect decision.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03028","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"cross-domain"},{"id":"bm_ldu-bench_c9c6a21a","familyId":"bmf_3b6629c4eda5","name":"LDU-Bench","oneLine":"LDU-Bench evaluates multimodal LLMs on lithography defect understanding through four tasks: defect triage, morphology recognition, coarse localization, and image-conditioned cause analysis, using task-level metrics and the Lithography Closure Score.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03078","pdf":"https://arxiv.org/pdf/2608.03078","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03078"},"evidence":{"snippet":"To this end, this paper proposes LDU-Bench, a multi-task multimodal benchmark for lithography defect understanding.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03078"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LDU-Bench evaluates multimodal LLMs on lithography defect understanding through four tasks: defect triage, morphology recognition, coarse localization, and image-conditioned cause analysis, using task-level metrics and the Lithography Closure Score.","whyItMatters":"Existing industrial anomaly detection benchmarks focus on defect presence, but lithography review requires deeper understanding of morphology, location, and causes. This benchmark aims to quantify those capabilities in a unified platform.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"74676a0275c2c257e1187afb95402961fe5bdd89836397d35e084d00d13f8fd7"},"motivation":"Multimodal large language models have demonstrated strong defect recognition capability in industrial anomaly detection.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03078","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_geb-bench_e0eb6ac0","familyId":"bmf_d4f2e362a4ae","name":"GEB-Bench","oneLine":"GEB-Bench evaluates models on abstract structural motifs (e.g., self-reference, strange loops) presented in multiple modalities (natural scenes, folk stories, math theorems, programmatic skeletons) with tasks probing cross-modal mapping.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04111","pdf":"https://arxiv.org/pdf/2608.04111","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.04111"},"evidence":{"snippet":"We introduce GEB-Bench, a benchmark whose unit is an abstract structural motif--self-reference, a strange loop, a Mobius twist--in the spirit of Godel, Escher, Bach.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04111"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GEB-Bench evaluates models on abstract structural motifs (e.g., self-reference, strange loops) presented in multiple modalities (natural scenes, folk stories, math theorems, programmatic skeletons) with tasks probing cross-modal mapping.","whyItMatters":"The benchmark aims to measure abstraction and cross-modal transfer, which are foundational for general intelligence but not addressed by current benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"899f4d4c01ea08d02222f773e78e95652c08b561e83b505fb17cc433c106a6e1"},"motivation":"Can a model look at a river delta and a lightning bolt and see that they share a structure?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04111","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_gdpevo_5e650b40","familyId":"bmf_7fbe6f3635be","name":"GDPevo","oneLine":"GDPevo evaluates agent self-evolution on real business tasks across 24 groups (240 tasks) in domains like CRM, ERP, finance, and healthcare. It uses rule hybridization to attribute test-time gains to training experience, with held-out test tasks.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Self-Evolution","Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03764","pdf":"https://arxiv.org/pdf/2608.03764","project":null,"code":"https://github.com/Prism-Shadow/GDPevo","data":null,"hfPaper":"https://huggingface.co/papers/2608.03764"},"evidence":{"snippet":"We present GDPevo, an evolution-native benchmark grounded in GDP-related enterprise workflows, together with the fully automated data pipeline that generates it.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":27,"hfDailySubmittedAt":"2026-08-06T00:00:00.000Z","githubStars":59,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03764"},"ranking":{"30d":{"score":86,"rank":6,"coverage":0.85,"confidence":"High"},"90d":{"score":77,"rank":31,"coverage":0.7,"confidence":"Medium"}},"description":"GDPevo evaluates agent self-evolution on real business tasks across 24 groups (240 tasks) in domains like CRM, ERP, finance, and healthcare. It uses rule hybridization to attribute test-time gains to training experience, with held-out test tasks.","whyItMatters":"Existing benchmarks lack attribution of gains to training experience and face data contamination. This benchmark provides an automated pipeline for evolving benchmark instances and measures self-evolution ability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2d42f74ab15e994c8e08c218936059ee891d31c0d96df64449f638381afcabea"},"motivation":"Agent self-evolution updates an agent's persistent state from prior experience and reuses it to solve related tasks more effectively.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03764","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Prism-Shadow","organizationType":"community","sourceUrl":"https://github.com/Prism-Shadow/GDPevo","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_finverse_651ab4a4","familyId":"bmf_7c67cb28d840","name":"FinVerse","oneLine":"FinVerse is a finance-domain time-series forecasting benchmark with 116,897 series (171.1M observations), selecting 60,232 as evaluation targets. It defines 11 metric families (78 metrics) tailored to each series' economic meaning.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03259","pdf":"https://arxiv.org/pdf/2608.03259","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03259"},"evidence":{"snippet":"To this end, we introduce FinVerse, a finance-domain time-series forecasting benchmark that takes a first step toward more realistic evaluation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03259"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FinVerse is a finance-domain time-series forecasting benchmark with 116,897 series (171.1M observations), selecting 60,232 as evaluation targets. It defines 11 metric families (78 metrics) tailored to each series' economic meaning.","whyItMatters":"Generic forecasting benchmarks use uniform error metrics that may not align with financial decision objectives. This benchmark aims to provide domain-aware evaluation that better reflects real-world utility.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a3fa652ffd2bb2f057ac9fad57ec414ff36f1d19038a8b0af1f073749a2628d7"},"motivation":"As time-series foundation models have emerged, the need for benchmarks that can evaluate their forecasting ability in meaningful ways has become increasingly important.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03259","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_finprobench_b6fc7343","familyId":"bmf_72a8e0c3d342","name":"FinProBench","oneLine":"FinProBench evaluates financial AI agents on professional tasks using rubrics derived from practitioner deliverables. It includes 1,723 curated deliverables across 57 occupations and an evaluation set of 20 tasks.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04077","pdf":"https://arxiv.org/pdf/2608.04077","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.04077"},"evidence":{"snippet":"We introduce FinProBench, a benchmark for professional financial tasks, and Role-Grounded Rubric Construction (RGRC), a reusable pipeline that derives rubrics from deliverables produced by practitioners in the same role.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04077"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FinProBench evaluates financial AI agents on professional tasks using rubrics derived from practitioner deliverables. It includes 1,723 curated deliverables across 57 occupations and an evaluation set of 20 tasks.","whyItMatters":"The benchmark addresses the need for evaluating financial AI agents against professional standards, offering an approach to derive rubrics from expert deliverables, which can reduce effort in rubric construction.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"33148f289e9f7ae843e6eaa2b9da7aa1f7aff37a902bb011593bdd36cc88e9bf"},"motivation":"Evaluating financial AI agents requires criteria aligned with real professional work.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04077","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_finperma_25dab076","familyId":"bmf_02e40c205e8e","name":"FinPerMA","oneLine":"FinPerMA evaluates personalized memory in LLM agents using frozen longitudinal investor trajectories with theory-informed impact rules. It includes a Post-Shock checkpoint to assess integration of material events into persistent user models, with 2,994 questions from 276 personas.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04095","pdf":"https://arxiv.org/pdf/2608.04095","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.04095"},"evidence":{"snippet":"We introduce FinPerMA, an event-grounded benchmark that evaluates personalized memory against frozen longitudinal investor trajectories.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04095"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FinPerMA evaluates personalized memory in LLM agents using frozen longitudinal investor trajectories with theory-informed impact rules. It includes a Post-Shock checkpoint to assess integration of material events into persistent user models, with 2,994 questions from 276 personas.","whyItMatters":"Existing personalized-memory benchmarks lack event-driven preference adaptation. FinPerMA fills this gap by providing an event-grounded evaluation, enabling assessment of whether agents can update user models over long horizons in high-stakes financial advising contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"42a3b21e6d5f636483446b11cbadd8f9a8a5b2891695f15cae3f512dfd5bd0a8"},"motivation":"Large language model (LLM) agents are increasingly used as personalized assistants in high-stakes domains such as financial advising, yet it remains unclear whether they can maintain and update an individualized user model over long horizons.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04095","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_fakei2v-bench_02c67b9d","familyId":"bmf_c3f4d16ac3a8","name":"FakeI2V-Bench","oneLine":"FakeI2V-Bench is a benchmark for evaluating image-level and video-level deepfake detectors in video detection scenarios. It comprises 97,548 videos generated by recent generation models and covers multiple content categories, with detectors scored by AUC on detection tasks.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CR"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.03096","pdf":"https://arxiv.org/pdf/2608.03096","project":null,"code":"https://github.com/CryptoAILab/FakeI2V-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.03096"},"evidence":{"snippet":"To fill this gap, we present FakeI2V-Bench, a benchmark for evaluating state-of-the-art video-level deepfake detectors in challenging scenarios, with a particular focus on systematically assessing the performance of image-level deepfake detectors in the video domain.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03096"},"ranking":{"30d":{"score":56,"rank":53,"coverage":0.55,"confidence":"Low"},"90d":{"score":44,"rank":243,"coverage":0.55,"confidence":"Low"}},"description":"FakeI2V-Bench is a benchmark for evaluating image-level and video-level deepfake detectors in video detection scenarios. It comprises 97,548 videos generated by recent generation models and covers multiple content categories, with detectors scored by AUC on detection tasks.","whyItMatters":"Existing deepfake video benchmarks lack evaluation of image-level detectors' transferability to video, and this benchmark provides a large-scale, standardized protocol to measure detector performance across both detector types, aiding in model selection and development for practical deepfake video detection.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4cbcb3e1a19c4006c5a3aa6c842acbd54bcf5a0e28aed8a1d803354198010ff9"},"motivation":"Recent advances in video generation models have significantly intensified the deepfake threat, yet the current deepfake video detection benchmarks remain underdeveloped.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"KDD 2026, Jeju, Korea, August 9-13, 2026","evidence":"To Appear in KDD 2026, Jeju, Korea, August 9-13, 2026","evidenceUrl":"https://arxiv.org/abs/2608.03096","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"KDD 2026, Jeju, Korea, August 9-13, 2026","reviewStatus":"accepted","decisionRaw":"To Appear in KDD 2026, Jeju, Korea, August 9-13, 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.03096","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"To Appear in KDD 2026, Jeju, Korea, August 9-13, 2026","level":"author-claim"}]}],"publishers":[{"name":"CryptoAILab","organizationType":"academic-lab","sourceUrl":"https://github.com/CryptoAILab/FakeI2V-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_educlaw-bench_d4a7a280","familyId":"bmf_9d60a04e1c2d","name":"EduClaw-Bench","oneLine":"EduClaw-Bench evaluates pedagogical LLM agents in a simulated 30-day tutoring relationship with a knowledge-tracing-based simulated learner, scoring learning gain, responsiveness, helpfulness, and curriculum design across 55 scenarios.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03206","pdf":"https://arxiv.org/pdf/2608.03206","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03206"},"evidence":{"snippet":"We introduce EduClaw-Bench, a benchmark that places an agent tutor in a continuous 30-day relationship with a simulated learner grounded in knowledge tracing (KT), whose knowledge-concept mastery, from a KT model trained on real-student data, drives its answers and is probed for learning gain across 55 scenarios.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03206"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EduClaw-Bench evaluates pedagogical LLM agents in a simulated 30-day tutoring relationship with a knowledge-tracing-based simulated learner, scoring learning gain, responsiveness, helpfulness, and curriculum design across 55 scenarios.","whyItMatters":"Existing benchmarks focus on single-turn tasks, leaving long-horizon tutoring unmeasured. This benchmark provides a standardized way to assess sustained pedagogical interaction, helping developers and educators choose agents that maintain effective teaching over time.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a7b659278f29aba313d02677f5db56cb5adb8ef8455ff8b3d47b1f457ab078fb"},"motivation":"Large language models (LLMs) power educational applications from tutoring to essay scoring, but each is a point solution to a single task, and only recently have these point solutions been integrated into agents operating over a learning management system (LMS).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03206","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_diagchain_064ef8b0","familyId":"bmf_818fcbc7cc94","name":"DiagChain","oneLine":"DiagChain evaluates LLM agents on evidence-grounded attack chain reconstruction from heterogeneous telemetry. It provides 69 scenarios and five metrics for stage-wise assessment of reasoning steps.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03591","pdf":"https://arxiv.org/pdf/2608.03591","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03591"},"evidence":{"snippet":"We present DiagChain, a diagnostic benchmark for evidence-grounded attack chain reconstruction that enables stage-wise evaluation of LLM agents.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03591"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DiagChain evaluates LLM agents on evidence-grounded attack chain reconstruction from heterogeneous telemetry. It provides 69 scenarios and five metrics for stage-wise assessment of reasoning steps.","whyItMatters":"Existing benchmarks often report end-to-end accuracy, hiding where errors arise. DiagChain enables systematic diagnosis of intermediate stages, offering actionable insights for improving cybersecurity agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cee8f4728507ea98f957ed73f00e42d1816a5d20d4530da2c45f8ed06d2adf58"},"motivation":"Large Language Model (LLM) agents offer a promising approach to attack chain reconstruction by retrieving and interpreting heterogeneous telemetry to infer ordered attacker actions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03591","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_dblifebench_c72360a2","familyId":"bmf_9c76077fbb9b","name":"DBLifeBench","oneLine":"Evaluates LLMs across five database lifecycle phases using tasks like schema design, SQL implementation, and debugging, with a focus on holistic database management capabilities.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.DB"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03794","pdf":"https://arxiv.org/pdf/2608.03794","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03794"},"evidence":{"snippet":"To bridge this gap, we introduce DBLifeBench, the first benchmark to evaluate LLMs across five critical lifecycle phases: Design, Implementation, Operation, Debugging, and Maintenance.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03794"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates LLMs across five database lifecycle phases using tasks like schema design, SQL implementation, and debugging, with a focus on holistic database management capabilities.","whyItMatters":"Moves beyond Text-to-SQL to cover the full database lifecycle, revealing trade-offs between specialized and general models and supporting full-stack database intelligence.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"983637e55ee018e991c08d341856fe784b46e695c3ea5eadd94f480a5160d9e2"},"motivation":"Large Language Models (LLMs) are transforming database interaction paradigms, evolving from simple query translators to autonomous database administrators (DBAs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03794","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DBLifeBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.03794","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_dataspace_1846702d","familyId":"bmf_2f3146cd78ab","name":"DataSpace","oneLine":"DataSpace evaluates data agents on analytical questions over heterogeneous workspaces containing CSV, JSON, SQLite, Markdown, PDF, and video artifacts. Agents must discover and integrate relevant evidence and return complete tabular results, scored by a deterministic evaluator using header-invariant column alignment and type-aware comparison.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03451","pdf":"https://arxiv.org/pdf/2608.03451","project":null,"code":"https://github.com/HKUSTDial/DataSpace","data":null,"hfPaper":"https://huggingface.co/papers/2608.03451"},"evidence":{"snippet":"We introduce DataSpace, a benchmark in which data agents produce verifiable tabular results from task-local heterogeneous workspaces.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":33,"hfDailySubmittedAt":"2026-08-07T00:00:00.000Z","githubStars":24,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03451"},"ranking":{"30d":{"score":83,"rank":8,"coverage":0.85,"confidence":"High"},"90d":{"score":71,"rank":70,"coverage":0.7,"confidence":"Medium"}},"description":"DataSpace evaluates data agents on analytical questions over heterogeneous workspaces containing CSV, JSON, SQLite, Markdown, PDF, and video artifacts. Agents must discover and integrate relevant evidence and return complete tabular results, scored by a deterministic evaluator using header-invariant column alignment and type-aware comparison.","whyItMatters":"Existing benchmarks isolate structured querying, retrieval, or open-ended analysis. DataSpace unifies evidence discovery, tabular output completeness, and deterministic evaluation, addressing a gap in assessing data agents for real-world analytical tasks. It provides a practical decision value for comparing agent harnesses and multimodal backbones.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"85b53d1fa3d535a851c3dc6cdf57e78343e5aad0a97b9882faeb637accaf314f"},"motivation":"Data agents enable natural-language analytics over organizational workspaces, where relevant evidence may be scattered across databases, structured files, long documents, and multimedia.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03451","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HKUST Dial","organizationType":"academic-lab","sourceUrl":"https://github.com/HKUSTDial/DataSpace","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_conlangbench_b9af66c3","familyId":"bmf_fd59187772ea","name":"ConlangBench","oneLine":"ConlangBench evaluates large language models on translation and vocabulary learning across 21 constructed languages, with a corpus of over 21 million conlang-English parallel sentence pairs and 321K vocabulary entries. The benchmark includes bidirectional translation tasks and learning-curve analysis for models trained on eight conlangs with sufficient parallel data.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.03505","pdf":"https://arxiv.org/pdf/2608.03505","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03505"},"evidence":{"snippet":"We present ConlangBench, the first large-scale benchmark for evaluating and training LLMs on 21 existing conlangs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03505"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ConlangBench evaluates large language models on translation and vocabulary learning across 21 constructed languages, with a corpus of over 21 million conlang-English parallel sentence pairs and 321K vocabulary entries. The benchmark includes bidirectional translation tasks and learning-curve analysis for models trained on eight conlangs with sufficient parallel data.","whyItMatters":"ConlangBench addresses the gap in evaluating LLMs on low-resource languages with diverse linguistic structures. It provides a controlled testbed for studying language acquisition and cross-lingual transfer, offering practical insights for model developers targeting underrepresented languages.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cc778933dc9f0238630f3713d79065c4088d322a404a6dd081accf04595e492c"},"motivation":"Constructed languages (conlangs) are intentionally created human languages with a rich tradition of linguistic creativity.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03505","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ConlangBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.03505","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_comex_9077da26","familyId":"bmf_34d8cbd69dec","name":"COMEX","oneLine":"COMEX is a benchmark for explainable aesthetic image cropping containing 33,161 quadruples of expanded image, crop box, composition category, and composition-grounded explanation. It supports joint evaluation of crop localization, composition understanding, and explanation generation.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07570","pdf":"https://arxiv.org/pdf/2608.07570","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07570"},"evidence":{"snippet":"To support this setting, we introduce COMEX, a new benchmark built through image expansion and an IO-reversal pipeline.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07570"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"COMEX is a benchmark for explainable aesthetic image cropping containing 33,161 quadruples of expanded image, crop box, composition category, and composition-grounded explanation. It supports joint evaluation of crop localization, composition understanding, and explanation generation.","whyItMatters":"Provides a structured evaluation of crop-and-explain methods, enabling comparison across models on composition-grounded reasoning and explanation generation, addressing the gap of evaluating explainability in image cropping.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"778832c8b2ffa61e32f6c420cf2ecdbb70d67cbeb9fdd63b3a78b3fc98a3f1ed"},"motivation":"Explainable aesthetic image cropping requires not only localizing a visually pleasing crop but also explaining why it is preferred.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07570","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_codeassay_bb1ca7aa","familyId":"bmf_30e11e735fce","name":"CodeAssay","oneLine":"CodeAssay is a benchmark of 185 Python tasks across ten software-engineering categories with audited ground truth, public and hidden tests, and code-property measures. It evaluates LLM code generation correctness and other code properties.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation"],"topics":["Code"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03535","pdf":"https://arxiv.org/pdf/2608.03535","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03535"},"evidence":{"snippet":"We present CodeAssay, a taxonomy-first benchmark of 185 Python tasks across ten software-engineering categories.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03535"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CodeAssay is a benchmark of 185 Python tasks across ten software-engineering categories with audited ground truth, public and hidden tests, and code-property measures. It evaluates LLM code generation correctness and other code properties.","whyItMatters":"Provides a reproducible basis for evaluating LLM-generated code with validated ground truth and multiple metrics, enabling evidence-based model selection in software development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f1d010099065690cca44a1f81bf2420f8a06c01f23ebde611e338cbdde5c3228"},"motivation":"Large Language Models are increasingly evaluated for code generation using test-based benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03535","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_chartanno_e949f852","familyId":"bmf_41b04716eec5","name":"ChartAnno","oneLine":"ChartAnno is a benchmark for evaluating MLLMs on chart annotation generation with 1,200 real-world charts and paired code/instructions. It evaluates annotation quality under different instruction specificity and input settings.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03464","pdf":"https://arxiv.org/pdf/2608.03464","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03464"},"evidence":{"snippet":"To address this gap, we introduce ChartAnno, a benchmark for evaluating MLLMs on chart annotation generation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03464"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ChartAnno is a benchmark for evaluating MLLMs on chart annotation generation with 1,200 real-world charts and paired code/instructions. It evaluates annotation quality under different instruction specificity and input settings.","whyItMatters":"Addresses the underexplored task of chart annotation, providing a testbed for comparing MLLM capabilities in generating communicative chart annotations.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fdc790effa5e4f1c49529a75ccc99adabe8ff5558f229a21857d2d4ae760c553"},"motivation":"Multimodal large language models (MLLMs) have made significant progress in chart understanding, generation, and editing, but their ability to annotate existing charts remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03464","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_care-bench_02aa6f70","familyId":"bmf_1e30cd79e194","name":"CARE-Bench","oneLine":"CARE-Bench is a source-grounded benchmark for patient-facing triage evaluation with 500 cases and 1,059 patient-disclosure prefixes. It evaluates LLMs on a four-label current-action task across 269 held-out rounds.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03731","pdf":"https://arxiv.org/pdf/2608.03731","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03731"},"evidence":{"snippet":"We introduce CARE-Bench, a source-grounded benchmark that evaluates sequential patient-facing triage as a four-label per-turn current-action task.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03731"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CARE-Bench is a source-grounded benchmark for patient-facing triage evaluation with 500 cases and 1,059 patient-disclosure prefixes. It evaluates LLMs on a four-label current-action task across 269 held-out rounds.","whyItMatters":"Provides a standardized evaluation for patient-facing triage, enabling comparison of models on action timing and clarification steps, critical for deployment safety.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"77d397efce78f8690a8abaa4ae9851616af81c5b4ba0f97f3ceb2bcc69c4977f"},"motivation":"Patient-facing medical LLMs and agents increasingly answer symptom questions before clinician contact, where the key safety question is what action the user should take next.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03731","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_brainbench_1fd81276","familyId":"bmf_4ec75908a077","name":"BrainBench","oneLine":"BrainBench is a benchmark for instruction-conditioned EEG understanding covering four subsets across 17 datasets. It evaluates LLMs on producing scientific reports and artifacts from EEG recordings.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.04156","pdf":"https://arxiv.org/pdf/2608.04156","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.04156"},"evidence":{"snippet":"We introduce \\benchmarkname{}, a unified benchmark for comprehensive, instruction-conditioned EEG understanding.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.04156"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"BrainBench is a benchmark for instruction-conditioned EEG understanding covering four subsets across 17 datasets. It evaluates LLMs on producing scientific reports and artifacts from EEG recordings.","whyItMatters":"Provides a unified evaluation for comprehensive EEG understanding, enabling comparison across models and execution paradigms, advancing LLM-based EEG analysis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ac297908643627604420be8cfe12469eecadbd7c1fe2abfc144b2a79e14b8442"},"motivation":"Electroencephalography (EEG) analysis extends beyond assigning predefined labels to recordings; it requires workflows connecting natural-language instructions, signal processing, quantitative evidence, and scientific interpretation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.04156","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_banglawild_bb99b93a","familyId":"bmf_14ca929f6ebd","name":"BanglaWild","oneLine":"BanglaWild is a benchmark of 2,535 Bengali scene text images with verbatim gold transcriptions and diagnostic attributes. It evaluates OCR and vision-language models on in-the-wild scene text recognition.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03884","pdf":"https://arxiv.org/pdf/2608.03884","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03884"},"evidence":{"snippet":"To address this gap, we introduce BANGLAWILD, a benchmark of 2,535 Bengali scene text images, each paired with a verbatim gold transcription, two categorical axes, four diagnostic attributes, and an orthographically standard form where the in-image text deviates from canonical spelling.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03884"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"BanglaWild is a benchmark of 2,535 Bengali scene text images with verbatim gold transcriptions and diagnostic attributes. It evaluates OCR and vision-language models on in-the-wild scene text recognition.","whyItMatters":"Fills the gap of measuring in-the-wild Bengali scene text recognition, providing a common testbed for OCR and VLMs and enabling analysis of error types.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"310fbe67f6a81ed656d1d8417f49aba8997fa40d7fb29735aa05d8a0c8d04162"},"motivation":"In-the-wild Bengali scene text recognition is largely unmeasured: existing resources target handwritten documents or constrained sign-board parsing, report only aggregate edit-distance metrics, and evaluate either conventional OCR or VLMs, never both on the same in-the-wild data.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03884","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_arteculture_b1d1e3cf","familyId":"bmf_ced67ccbede0","name":"ArtECulture","oneLine":"ArtECulture is a benchmark for culture-conditioned visual emotion understanding with 6,792 artworks and culture-specific labels across three cultures. It evaluates MLLMs on predicting cultural emotion perception and explaining rationale.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03358","pdf":"https://arxiv.org/pdf/2608.03358","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03358"},"evidence":{"snippet":"Thus, we present ArtECulture, a benchmark containing 6,792 artworks with culture-specific emotion labels and explanations across English, Chinese, and Arabic cultures, with balanced Western and non-Western content.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03358"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ArtECulture is a benchmark for culture-conditioned visual emotion understanding with 6,792 artworks and culture-specific labels across three cultures. It evaluates MLLMs on predicting cultural emotion perception and explaining rationale.","whyItMatters":"Provides a balanced benchmark for evaluating cultural variations in visual emotion understanding, enabling comparison of MLLM capabilities across cultures.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"26e56b24950b033194ffdcc47620298b9868078a1bc1ee3b29c58b972e83a34e"},"motivation":"Existing visual emotion understanding methods typically ignore cultural variations in emotional perception.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03358","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_antiskillbench_e40657d8","familyId":"bmf_cdeea9a0b393","name":"AntiSkillBench","oneLine":"AntiSkillBench is a benchmark for evaluating privacy leakage and impersonation risks in persona-skills. It includes 7,500 persona-grounded dialogue traces and evaluates across three skill-distillation strategies.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.03700","pdf":"https://arxiv.org/pdf/2608.03700","project":"https://yonglixiang.github.io/AntiSkillBench","code":"https://github.com/yonglixiang/AntiSkillBench","data":null,"hfPaper":"https://huggingface.co/papers/2608.03700"},"evidence":{"snippet":"To systematically investigate the safety of the persona-skill pipeline, we introduce AntiSkillBench, an end-to-end benchmark for evaluating risks and defenses across the persona-skill pipeline.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":9,"hfDailySubmittedAt":"2026-08-05T00:00:00.000Z","githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03700"},"ranking":{"30d":{"score":54,"rank":57,"coverage":0.85,"confidence":"High"},"90d":{"score":42,"rank":254,"coverage":0.7,"confidence":"Medium"}},"description":"AntiSkillBench is a benchmark for evaluating privacy leakage and impersonation risks in persona-skills. It includes 7,500 persona-grounded dialogue traces and evaluates across three skill-distillation strategies.","whyItMatters":"Provides an end-to-end evaluation of risks and defenses in the persona-skill pipeline, enabling comparison of model safety across different settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b964d761566aee8f471a2d53c607fb5abc799cb8ebc4c0b781ff6c7c4c9d99f9"},"motivation":"Persona skills distill personal interaction histories into portable and executable artifacts for downstream agents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03700","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Yongli Xiang","organizationType":"academic-lab","sourceUrl":"https://yonglixiang.github.io/AntiSkillBench","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_ai-security-leaderboard_0982a1ca","familyId":"bmf_ee1e051ce1cb","name":"AI Security Leaderboard","oneLine":"The AI Security Leaderboard evaluates frontier AI model safeguards against the FAR.AI Minimal Standard for Safeguards across severe misuse requests (CBRNE threats and offensive cybersecurity), testing for universal jailbreaks.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.03070","pdf":"https://arxiv.org/pdf/2608.03070","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.03070"},"evidence":{"snippet":"The AI Security Leaderboard is an independent benchmark that ranks the safeguards of frontier AI models from least to most secure.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.03070"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"The AI Security Leaderboard evaluates frontier AI model safeguards against the FAR.AI Minimal Standard for Safeguards across severe misuse requests (CBRNE threats and offensive cybersecurity), testing for universal jailbreaks.","whyItMatters":"Provides a public, rolling comparison of model security that quantifies cost to jailbreak and reveals large gaps, informing deployment decisions for high-risk capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"69ac3a8722b5e6cde249e6d296b4e83ea963c38877341273c83327b2b789d587"},"motivation":"The AI Security Leaderboard is an independent benchmark that ranks the safeguards of frontier AI models from least to most secure.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.03070","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_agentchaosbench_3680a518","familyId":"bmf_6edc4728dc4b","name":"AGENTCHAOSBENCH","oneLine":"AGENTCHAOSBENCH is a dataset of sanitized execution traces from five agentic applications with injected runtime faults, used to evaluate fault detection and localization from telemetry.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.14680","pdf":"https://arxiv.org/pdf/2608.14680","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.14680"},"evidence":{"snippet":"We present AGENTCHAOSBENCH, a benchmark for detecting and localizing runtime faults in agentic systems from their execution telemetry.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.14680"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AGENTCHAOSBENCH is a dataset of sanitized execution traces from five agentic applications with injected runtime faults, used to evaluate fault detection and localization from telemetry.","whyItMatters":"Addresses runtime fault diagnosis in LLM agentic systems, offering a reproducible task for comparing diagnostic methods across tool and agent boundaries.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d8265f900a0f5fb1b712b381254c06eae813c895964a23a9b7412f07c0e9c5c4"},"motivation":"Reliability in LLM-based agentic systems is a property of the whole execution (its tool calls, model calls, guardrails, and inter-agent messages), not of the final answer alone, yet evaluating only task outcomes reveals little about how or why a run fails.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.14680","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_worldexam_7f2f166e","familyId":"bmf_3931c2ca2952","name":"WorldExam","oneLine":"WorldExam evaluates controllable video generation models as world models across four levels: Visual Quality, Control Adherence, Spatial Consistency, and World Reactivity, with 1,474 cases across eight tasks.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02603","pdf":"https://arxiv.org/pdf/2608.02603","project":"https://WorldExam.github.io","code":"https://github.com/YuxueYang1204/worldexam","data":null,"hfPaper":"https://huggingface.co/papers/2608.02603"},"evidence":{"snippet":"We introduce WorldExam, a hierarchical diagnostic benchmark spanning four levels: Visual Quality, Control Adherence, Spatial Consistency, and World Reactivity.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":34,"hfDailySubmittedAt":"2026-08-04T00:00:00.000Z","githubStars":21,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02603"},"ranking":{"30d":{"score":83,"rank":10,"coverage":0.85,"confidence":"High"},"90d":{"score":69,"rank":81,"coverage":0.7,"confidence":"Medium"}},"description":"WorldExam evaluates controllable video generation models as world models across four levels: Visual Quality, Control Adherence, Spatial Consistency, and World Reactivity, with 1,474 cases across eight tasks.","whyItMatters":"Fills the gap in evaluating inherent world reactivity beyond visual quality and instruction fulfillment, providing a hierarchical diagnostic for model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ad6f7e1c7fa7d00cdeb16255b1471a37c16e556e55fa1f1cef5b11b412f7da0e"},"motivation":"Controllable video generation models are increasingly being developed as world models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02603","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_uav3dcrop_4b4a17ef","familyId":"bmf_fbd68110137b","name":"UAV3DCrop","oneLine":"UAV3DCrop is a benchmark of repeated multi-angle UAV crop surveys with 88,830 images across 91 scenes, evaluating 3D reconstruction methods on appearance, geometry, and canopy height.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.06404","pdf":"https://arxiv.org/pdf/2608.06404","project":"https://link-dev.github.io/UAV3DCrop/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.06404"},"evidence":{"snippet":"We introduce UAV3DCrop, a public benchmark of repeated multi-angle unmanned aerial vehicle (UAV) crop surveys.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.06404"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"UAV3DCrop is a benchmark of repeated multi-angle UAV crop surveys with 88,830 images across 91 scenes, evaluating 3D reconstruction methods on appearance, geometry, and canopy height.","whyItMatters":"Provides a domain-specific benchmark for agronomically relevant 3D reconstruction, revealing that generic methods do not directly translate to crop monitoring.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9658c34d4f7bdd6619c069d200edba2f4f07787bf6607b4e2a0a30e0eb76b7db"},"motivation":"Accurate 3D crop monitoring underpins data-driven precision agriculture by enabling field-scale analysis of plant structure, growth dynamics, and management response.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.06404","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_svgeval_0c4e294c","familyId":"bmf_a28c66cbe961","name":"SVGEval","oneLine":"SVGEval is a vision-grounded benchmark for human-aligned SVG quality assessment, with multi-round labeled annotations, evaluating models on semantic, aesthetic, geometry, and layout judgments.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.01977","pdf":"https://arxiv.org/pdf/2608.01977","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01977"},"evidence":{"snippet":"We introduce SVGEval, a vision-grounded multimodal benchmark for human-aligned SVG quality assessment.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01977"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SVGEval is a vision-grounded benchmark for human-aligned SVG quality assessment, with multi-round labeled annotations, evaluating models on semantic, aesthetic, geometry, and layout judgments.","whyItMatters":"Addresses the lack of human-aligned evaluation for text-to-SVG generation, providing a reliable testbed for model comparison and improvement.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9f6dd5fb28a868e8d36d06bae162945d737e30f67a2293afb10dd65fd75f854e"},"motivation":"Multimodal large models are increasingly used to generate scalable vector graphics (SVG), but reliable evaluation remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026","evidence":"Accepted by ECCV 2026","evidenceUrl":"https://arxiv.org/abs/2608.01977","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026","reviewStatus":"accepted","decisionRaw":"Accepted by ECCV 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.01977","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by ECCV 2026","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_spike-bench_3b80754c","familyId":"bmf_71b5d4d6ae47","name":"SPIKE-Bench","oneLine":"SPIKE-Bench evaluates LLM biosecurity risks using 631 curated toxin-design prompts and a three-stage funnel (compliance, plausibility, predicted toxicity) producing the Functional Harmfulness Rate.","area":"Safety & Trustworthiness","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Safety"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02684","pdf":"https://arxiv.org/pdf/2608.02684","project":null,"code":"https://github.com/PKU-Alignment/SPIKE-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.02684"},"evidence":{"snippet":"To address this evaluation blind spot, we introduce SPIKE-Bench, coupling 631 curated toxin-design prompts across seven functional categories with the SPIKE funnel, a three-stage protocol that filters output through compliance, biological plausibility, and predicted toxicity, producing stage-level diagnostics and an aggregate function-aware metric: the Functional Harmfulness Rate (FHR).","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02684"},"ranking":{"30d":{"score":50,"rank":73,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":282,"coverage":0.55,"confidence":"Low"}},"description":"SPIKE-Bench evaluates LLM biosecurity risks using 631 curated toxin-design prompts and a three-stage funnel (compliance, plausibility, predicted toxicity) producing the Functional Harmfulness Rate.","whyItMatters":"Provides a function-aware metric for biosecurity evaluation beyond refusal rates, enabling comparison of models on predicted functional harm.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f5a2312fc8ce8ce06a37a24ed20b5009d784e622db5fd68cc48c0356f8266367"},"motivation":"Large Language Models (LLMs) are accelerating biological research, yet this same capability poses a critical biosecurity threat: models that assist in protein engineering can equally be prompted to generate predicted toxin-like sequences, potentially lowering the barrier to biological misuse.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"COLM 2026","evidence":"Accepted to COLM 2026. 40 pages, 9 figures","evidenceUrl":"https://arxiv.org/abs/2608.02684","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"COLM 2026","reviewStatus":"accepted","decisionRaw":"Accepted to COLM 2026. 40 pages, 9 figures","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.02684","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to COLM 2026. 40 pages, 9 figures","level":"author-claim"}]}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_scrambletoolbench_aa15193b","familyId":"bmf_2758dc7261ac","name":"ScrambleToolBench","oneLine":"ScrambleToolBench evaluates agent behavioral reasoning in an interactive terminal environment with obfuscated tools and dynamic challenges like mapping drift and stochastic failures.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02358","pdf":"https://arxiv.org/pdf/2608.02358","project":null,"code":"https://github.com/declare-lab/ScrambleToolBench","data":null,"hfPaper":"https://huggingface.co/papers/2608.02358"},"evidence":{"snippet":"To address this limitation, we introduce ScrambleToolBench, an interactive terminal benchmark designed to isolate behavioral reasoning.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":11,"hfDailySubmittedAt":"2026-08-04T00:00:00.000Z","githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02358"},"ranking":{"30d":{"score":67,"rank":34,"coverage":0.85,"confidence":"High"},"90d":{"score":53,"rank":180,"coverage":0.7,"confidence":"Medium"}},"description":"ScrambleToolBench evaluates agent behavioral reasoning in an interactive terminal environment with obfuscated tools and dynamic challenges like mapping drift and stochastic failures.","whyItMatters":"Isolates behavioral reasoning from prior knowledge, revealing gaps in agent adaptation and supporting comparison of agent architectures.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3e01134c43707be0ccc50816693ac98caa4b3846a76af61e086562584d3efb0f"},"motivation":"To operate robustly in open-world environments, autonomous agents should be able to infer the behavior of unfamiliar systems through interaction alone, even in the absence of documentation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02358","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_scistylebench_232be1cc","familyId":"bmf_087086a27a87","name":"SciStyleBench","oneLine":"SciStyleBench diagnoses stylistic bias in LLM-based idea evaluation through controlled stylistic perturbations, metrics, and a mitigation extractor.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.01666","pdf":"https://arxiv.org/pdf/2608.01666","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01666"},"evidence":{"snippet":"To address this question, we propose SciStyleBench, a unified three-component benchmark for diagnosing and mitigating stylistic bias in LLM-based idea evaluation: (i) First, SciStyleStage, a three-stage evaluation environment that applies controlled stylistic perturbations to fixed scientific content across three settings no context, fixed-domain context, and open-domain retrieval context, covering 600 scientific ideas and 15 style variants, with 9,000 evaluation instances per setting; (ii) Seco","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01666"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SciStyleBench diagnoses stylistic bias in LLM-based idea evaluation through controlled stylistic perturbations, metrics, and a mitigation extractor.","whyItMatters":"Highlights the importance of style invariance in scientific idea evaluation and provides metrics and a mitigation module for improving LLM judges.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c5fa9ae335618287682507905754382d6f7077a6f33702e05d89bf2c1e48151a"},"motivation":"However, whether these judges truly evaluate the scientific substance of ideas or are influenced by superficial stylistic presentation remains an open question.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.01666","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness","Search & Retrieval"],"domainScope":"general"},{"id":"bm_prweaver_3962c935","familyId":"bmf_064f6a3de339","name":"PRWeaver","oneLine":"PRWeaver evaluates LLM-based code auditors against malicious pull requests across repository history, using 208 attacks and multiple renderings.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02693","pdf":"https://arxiv.org/pdf/2608.02693","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.02693"},"evidence":{"snippet":"We introduce PRWeaver, a benchmark of 208 execution-validated attacks from ten real-world repositories, each instantiated under four matched review renderings (832 renderings in total).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02693"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PRWeaver evaluates LLM-based code auditors against malicious pull requests across repository history, using 208 attacks and multiple renderings.","whyItMatters":"Addresses reliability of code auditing agents under adversarial conditions, informing deployment of such systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a617830945e0073bc1f8afcd65e7f8d0d1b6fc97c53ab1af077a5f10f3d5adfe"},"motivation":"LLM-based code auditors are increasingly integrated into pull-request (PR) workflows, yet their reliability against adversarial changes distributed across repository evolution remains poorly understood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02693","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_predact-bench_f35f1a5f","familyId":"bmf_9ef627ddae02","name":"PredAct-Bench","oneLine":"PredAct-Bench evaluates dialogue agents paired with imperfect tools using educational datasets, measuring AI-assisted decision-making and trust metrics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02372","pdf":"https://arxiv.org/pdf/2608.02372","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.02372"},"evidence":{"snippet":"We introduce PREDACTBENCH, a benchmark for evaluating dialogue agents paired with statistically imperfect tools, using education as a measurable testbed where ground truth outcomes and clear intervention decisions are available.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02372"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PredAct-Bench evaluates dialogue agents paired with imperfect tools using educational datasets, measuring AI-assisted decision-making and trust metrics.","whyItMatters":"Highlights gap in existing benchmarks that assume perfect tool reliability, important for high-stakes domains.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5aa412499027fa3711c2385a3edf655b6ddc1e651674a4a81be79947ca4f7618"},"motivation":"Large Language Models (LLMs) are increasingly deployed in task-oriented dialogue systems that support multi-step decision-making in high-stakes domains such as education, healthcare, and finance.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02372","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_onepot-bench_ffd24571","familyId":"bmf_ddcd7659e7a2","name":"onepot-Bench","oneLine":"onepot-Bench 0 is a proprietary benchmark suite evaluating language models on synthetic chemistry capabilities, including cheminformatics literacy, safety behavior, and reaction outcome prediction.","area":"Language & Knowledge","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals"],"capabilities":[],"topics":["cs.LG"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02595","pdf":"https://arxiv.org/pdf/2608.02595","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.02595"},"evidence":{"snippet":"We introduce onepot-Bench 0, a proprietary benchmark suite for evaluating language models on synthetic chemistry capabilities relevant to wet-lab execution.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02595"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"onepot-Bench 0 is a proprietary benchmark suite evaluating language models on synthetic chemistry capabilities, including cheminformatics literacy, safety behavior, and reaction outcome prediction.","whyItMatters":"Targets skills needed for reliable laboratory decisions, addressing limitations of public data benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"693005891e1b35ecdfaae1723d081526d7410b397c4f82f2556615eff7466b72"},"motivation":"Language models are playing an increasingly important role in laboratory science, performing tasks such as experiment planning, execution, and post-hoc analysis.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02595","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_medpress_3e59debc","familyId":"bmf_254263a6c1e2","name":"MedPRESS","oneLine":"MedPRESS evaluates LLM sycophancy in multi-turn medical dialogues, containing 600 five-turn scenarios across three families with structured judging and safety metrics.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02520","pdf":"https://arxiv.org/pdf/2608.02520","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.02520"},"evidence":{"snippet":"We introduce MedPRESS, a multi-turn benchmark for measuring patient-pressure-induced sycophancy in LLMs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02520"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MedPRESS evaluates LLM sycophancy in multi-turn medical dialogues, containing 600 five-turn scenarios across three families with structured judging and safety metrics.","whyItMatters":"Measures robustness under conversational pressure, a gap in static medical safety evaluations.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"27d4c7369310caf068cd25e943a2bf0b665f5ffa1f535771239004cb9edd6e24"},"motivation":"Large language models (LLMs) are increasingly used for health-related advice.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02520","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_ii-bench_6645329e","familyId":"bmf_e098118d24e6","name":"II-Bench","oneLine":"II-Bench evaluates computer-use agents against low-harm adversarial tasks across three platforms, with 444 examples and a testing framework.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02018","pdf":"https://arxiv.org/pdf/2608.02018","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.02018"},"evidence":{"snippet":"To systematically investigate this blind spot, we present II-Bench, a collection of seemingly harmless adversarial tasks.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02018"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"II-Bench evaluates computer-use agents against low-harm adversarial tasks across three platforms, with 444 examples and a testing framework.","whyItMatters":"Exposes security blind spots in human-in-the-loop defenses for computer-use agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7a0b6cccfe47f257d527480101c0136c923e632f86faa905e8e1e140fa873ce0"},"motivation":"Computer-use agents (CUAs), which empower large language models to autonomously operate operating systems and the web, are increasingly vulnerable to indirect prompt injection attacks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02018","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_iba-bench_e7767345","familyId":"bmf_cb530590be3a","name":"IBA-Bench","oneLine":"IBA-Bench evaluates LLM agents on implicit behavioral alignment using longitudinal interaction histories, across nine application domains.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02171","pdf":"https://arxiv.org/pdf/2608.02171","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.02171"},"evidence":{"snippet":"To address this challenge, we introduce IBA-Bench, a benchmark for implicit behavioral alignment constructed from longitudinal interaction histories that contain noise, implicit cues, and temporal inconsistencies.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02171"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"IBA-Bench evaluates LLM agents on implicit behavioral alignment using longitudinal interaction histories, across nine application domains.","whyItMatters":"Assesses whether agents can infer and satisfy implicit user constraints, a practical gap in personalization.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0a819d20cf1eb2a54f8a1067fdb7cfbc53be89103e7063a85298bb8749cca3d6"},"motivation":"Large Language Models have enabled increasingly capable autonomous agents, yet personalization remains critical for making such agents practically useful.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02171","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_gisagentbench_880052a5","familyId":"bmf_6d527eda7e33","name":"GISAgentBench","oneLine":"GISAgentBench evaluates LLM agents on multi-step GIS tasks from practitioner sources, with 349 tasks and executable reference trajectories.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.01645","pdf":"https://arxiv.org/pdf/2608.01645","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01645"},"evidence":{"snippet":"To address this gap, we introduce GISAgentBench, a benchmark of 349 multi-step GIS tasks curated from GIS Stack Exchange and instantiated on real public data across six selected geographic areas of interest.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01645"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GISAgentBench evaluates LLM agents on multi-step GIS tasks from practitioner sources, with 349 tasks and executable reference trajectories.","whyItMatters":"Provides deterministic evaluation with ground truth outputs, addressing limitations of surrogate signals.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"073a78b19cc0e94ac668cf91be2d364711fdd4c49dbf2626383828d202f64bdf"},"motivation":"Geographic Information System (GIS) professionals rely on multi-step spatial analysis workflows to support decision-making in urban planning, disaster response, and environmental monitoring.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.01645","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_geoid-flood_b555af53","familyId":"bmf_983fa02568b2","name":"GEOID-Flood","oneLine":"GEOID-Flood is a large-scale multi-modal benchmark for flood segmentation with over 14,000 tiles, co-registered SAR and optical data, and validated labels.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02315","pdf":"https://arxiv.org/pdf/2608.02315","project":null,"code":"https://github.com/links-ads/geoid-flood","data":null,"hfPaper":"https://huggingface.co/papers/2608.02315"},"evidence":{"snippet":"We introduce GEOID-Flood, a large-scale multi-modal flood segmentation benchmark, derived from Copernicus Emergency Management Service activations, spanning 219 events across 65 countries over ten years.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":"2026-08-04T00:00:00.000Z","githubStars":8,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02315"},"ranking":{"30d":{"score":65,"rank":39,"coverage":0.85,"confidence":"High"},"90d":{"score":57,"rank":153,"coverage":0.7,"confidence":"Medium"}},"description":"GEOID-Flood is a large-scale multi-modal benchmark for flood segmentation with over 14,000 tiles, co-registered SAR and optical data, and validated labels.","whyItMatters":"Supports evaluation of geospatial foundation models on flood mapping, enabling transfer studies.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f40b62e3c40d49e9dda267f58975c5d3aac5c567ea961885abece2e007bd3a70"},"motivation":"Geospatial foundation models aim to learn representations that transfer across regions and sensors, yet evaluating them on specific tasks requires large, high-quality, multi-modal benchmarks that measure how well such models extract value from data.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026 - Terrabytes II Workshop, 23 pages","evidence":"Accepted at ECCV 2026 - Terrabytes II Workshop, 23 pages","evidenceUrl":"https://arxiv.org/abs/2608.02315","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026 - Terrabytes II Workshop, 23 pages","reviewStatus":"accepted","decisionRaw":"Accepted at ECCV 2026 - Terrabytes II Workshop, 23 pages","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.02315","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at ECCV 2026 - Terrabytes II Workshop, 23 pages","level":"author-claim"}]}],"publishers":[{"name":"links-ads","organizationType":"academic-lab","sourceUrl":"https://github.com/links-ads/geoid-flood","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_gabench_d11355fd","familyId":"bmf_239d1223e7ea","name":"GABench","oneLine":"GABench evaluates LLM agents on graph analysis tasks across three graph types and four task categories: graph retrieval, graph theory, graph machine learning, and graph open-ended QA. It provides 84 executable tools and 10,400 tasks with verifiable ground truth.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.01684","pdf":"https://arxiv.org/pdf/2608.01684","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01684"},"evidence":{"snippet":"To address these limitations, we introduce GABench, a comprehensive benchmark for agentic graph analysis.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01684"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GABench evaluates LLM agents on graph analysis tasks across three graph types and four task categories: graph retrieval, graph theory, graph machine learning, and graph open-ended QA. It provides 84 executable tools and 10,400 tasks with verifiable ground truth.","whyItMatters":"Existing graph benchmarks lack coverage and typically format tasks as text QA, limiting agent evaluation. GABench offers a comprehensive, tool-based benchmark for assessing end-to-end agentic capabilities in graph analysis, providing practical insights into harness and tool-call quality.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4ac59933601519e111a1d238e66e2b720e26a7ffd9d23282afb3e6862415541c"},"motivation":"Large language model (LLM) agents are increasingly capable of planning, using tools, and interacting with external environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.01684","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_event-activitynet_532df77c","familyId":"bmf_f0108426e128","name":"Event ActivityNet","oneLine":"Event ActivityNet is a large-scale simulated-event benchmark for untrimmed action understanding, derived from ActivityNet videos. It includes 3,263 videos, 200 action classes, event-voxel representations, temporal annotations, and supports recognition, event-language alignment, and online temporal localization.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.01948","pdf":"https://arxiv.org/pdf/2608.01948","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01948"},"evidence":{"snippet":"We introduce Event ActivityNet, a large-scale simulated-event benchmark derived from human-annotated, untrimmed ActivityNet videos.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01948"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Event ActivityNet is a large-scale simulated-event benchmark for untrimmed action understanding, derived from ActivityNet videos. It includes 3,263 videos, 200 action classes, event-voxel representations, temporal annotations, and supports recognition, event-language alignment, and online temporal localization.","whyItMatters":"Existing datasets lack long-horizon event-based understanding. Event ActivityNet provides a scalable benchmark for long-horizon event modeling with multiple tasks, enabling evaluation of models on untrimmed action understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"07dbd75f826bdb9f61f3bc2b6b6b25b3f322c60be70ce8e799bd18c9ccfb5e47"},"motivation":"Long-horizon event-based action understanding remains underexplored because existing datasets largely comprise short, trimmed clips, while collecting native event streams with dense temporal annotations is costly.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.01948","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_culturevidbench_541d6118","familyId":"bmf_3a418ce01d2a","name":"CultureVidBench","oneLine":"CultureVidBench evaluates cultural understanding in text-to-video generation with 1,000 curated prompts covering 12 countries, 6 continents, and 14 cultural aspects. It assesses cultural faithfulness, multimodal cultural rendering, semantic adherence, and perceptual quality using human studies and MLLM-based automatic assessment.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.01942","pdf":"https://arxiv.org/pdf/2608.01942","project":"https://hanxjing.github.io/CultureVidBench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01942"},"evidence":{"snippet":"We introduce CultureVidBench, a comprehensive benchmark for evaluating cultural understanding in T2V generation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01942"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CultureVidBench evaluates cultural understanding in text-to-video generation with 1,000 curated prompts covering 12 countries, 6 continents, and 14 cultural aspects. It assesses cultural faithfulness, multimodal cultural rendering, semantic adherence, and perceptual quality using human studies and MLLM-based automatic assessment.","whyItMatters":"Existing T2V benchmarks focus on perceptual quality and alignment but neglect cultural representation. CultureVidBench addresses this gap by providing a benchmark for evaluating whether generated videos capture culturally specific details, which is crucial for diverse deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dfea5ff7823a68eef3254a2a732074b099e2660dc300cc21629a597a347f64b5"},"motivation":"Text-to-video (T2V) generation models have advanced rapidly, yet their ability to represent diverse cultural contexts remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.01942","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_confbench_a42d7b7f","familyId":"bmf_32ae0b47f61b","name":"ConfBench","oneLine":"ConfBench is a calibration-specific benchmark for key information extraction from documents. It applies 20 degradation pipelines to create 1,346 variants and over 70K entity-level evaluations, spanning the accuracy spectrum for evaluating confidence estimates of VLMs.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.01792","pdf":"https://arxiv.org/pdf/2608.01792","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01792"},"evidence":{"snippet":"We introduce ConfBench, the first calibration-specific benchmark for key information extraction (KIE), built by applying 20 controlled degradation pipelines to a diverse document set, yielding 1,346 variants and 70K+ entity-level evaluations spanning the full accuracy spectrum.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01792"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"ConfBench is a calibration-specific benchmark for key information extraction from documents. It applies 20 degradation pipelines to create 1,346 variants and over 70K entity-level evaluations, spanning the accuracy spectrum for evaluating confidence estimates of VLMs.","whyItMatters":"Document processing requires trustworthy confidence scores for routing automation vs. human review. ConfBench enables systematic study of confidence estimators and calibration methods, addressing the lack of calibration-focused benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"edd9b3b99ef0608d787b5bf639c8a47d44c6ea416927fcc3438f3972ec7d529e"},"motivation":"Intelligent document processing (IDP) with vision-language models (VLMs) hinges on confidence scores trustworthy enough to route extractions between automation and human review.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.01792","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_companionbench_b876ca73","familyId":"bmf_fa6e96451701","name":"CompanionBench","oneLine":"CompanionBench is an interactive bilingual benchmark for AI emotional companionship, grounding scenarios and a user simulator in de-identified real-world data. It evaluates ten capabilities derived from 25 theories, using a rubric and deterministic disclosure measure.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.02046","pdf":"https://arxiv.org/pdf/2608.02046","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.02046"},"evidence":{"snippet":"We introduce CompanionBench, an interactive bilingual benchmark.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02046"},"ranking":{"30d":{"score":47,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"CompanionBench is an interactive bilingual benchmark for AI emotional companionship, grounding scenarios and a user simulator in de-identified real-world data. It evaluates ten capabilities derived from 25 theories, using a rubric and deterministic disclosure measure.","whyItMatters":"LLM companions are deployed at scale but poorly evaluated. CompanionBench provides a reproducible, theory-anchored benchmark with real-world grounding, offering granular capability assessment and addressing judge biases.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"34da21590388dc063177ffd629f6d7635a793effad0796df39d51dba051bc8b8"},"motivation":"LLM companions are deployed at scale in personally consequential settings, yet poorly evaluated.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02046","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_bulkpr-bench_1253579f","familyId":"bmf_10e04fc9ee92","name":"BulkPR-Bench","oneLine":"BulkPR-Bench evaluates governance of interacting pull requests. It includes 581 candidate PRs on 18 repositories, with metrics RDS and Global-SGY measuring safe delivery. The benchmark uses executable repository execution with hidden safety checks.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02685","pdf":"https://arxiv.org/pdf/2608.02685","project":"https://doi.org/10.5281/zenodo.21717780","code":"https://github.com/Eureka246/BulkPR-Bench-Release","data":null,"hfPaper":"https://huggingface.co/papers/2608.02685"},"evidence":{"snippet":"We introduce BulkPR-Bench, an executable benchmark in which an agent must recover consequential PR relations and return a large safe subset in executable order under a rolling-release protocol.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02685"},"ranking":{"30d":{"score":34,"rank":134,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":357,"coverage":0.55,"confidence":"Low"}},"description":"BulkPR-Bench evaluates governance of interacting pull requests. It includes 581 candidate PRs on 18 repositories, with metrics RDS and Global-SGY measuring safe delivery. The benchmark uses executable repository execution with hidden safety checks.","whyItMatters":"Coding-agent benchmarks often assume independent PRs. BulkPR-Bench addresses interactive PR queues, providing a benchmark for evaluating joint decision-making in complex scenarios, with executable validation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"12c3924e5aee28a796c6e54f3bd0c7d13732d61f8f27783829c5fdb4b37cccb6"},"motivation":"Coding-agent benchmarks increasingly cover long-horizon, end-to-end, and interactive development, but typically retain one requested outcome or a fixed change sequence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02685","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Zenodo","organizationType":"benchmark-organization","sourceUrl":"https://doi.org/10.5281/zenodo.21717780","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_authmem-bench_56d03df5","familyId":"bmf_3b53359afe7a","name":"AuthMem-Bench","oneLine":"AuthMem-Bench evaluates authority collapse in persistent memory for LLM agents. It uses a paired benchmark holding claims and tasks fixed while varying source authority, measuring write-time collapse, authorization errors, and automatic authority preservation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Self-Evolution","Agents","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.01679","pdf":"https://arxiv.org/pdf/2608.01679","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01679"},"evidence":{"snippet":"We introduce AuthMem-Bench, a controlled paired benchmark that holds the focal claim and downstream task fixed while varying only source authority.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01679"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AuthMem-Bench evaluates authority collapse in persistent memory for LLM agents. It uses a paired benchmark holding claims and tasks fixed while varying source authority, measuring write-time collapse, authorization errors, and automatic authority preservation.","whyItMatters":"Memory consolidation can erase authority constraints, leading to unauthorized actions. AuthMem-Bench provides a controlled benchmark to measure and improve authority preservation in memory systems, relevant for safe agent deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"59f3b5e566de1dbc5839708d614c163fef369422470391756f0e748c095c89a6"},"motivation":"Persistent memory allows (self-evolving) LLM agents to adapt across tasks by consolidating heterogeneous interaction histories into reusable facts, preferences, observations, and rules.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.01679","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_schedbench_f5e0e993","familyId":"bmf_e3d434ea145f","name":"SCHEDBench","oneLine":"SCHEDBench evaluates LLM constraint faithfulness in natural-language combinatorial scheduling. It includes 1,132 instances across JSP, RCPSP, nurse rostering, and timetabling, with templated NL variations and solver-verified references.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00991","pdf":"https://arxiv.org/pdf/2608.00991","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00991"},"evidence":{"snippet":"This paper introduces SCHEDBench, a natural-language benchmark for evaluating combinatorial scheduling constraint faithfulness under surface-form variation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00991"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SCHEDBench evaluates LLM constraint faithfulness in natural-language combinatorial scheduling. It includes 1,132 instances across JSP, RCPSP, nurse rostering, and timetabling, with templated NL variations and solver-verified references.","whyItMatters":"LLMs must generate constraint-feasible schedules under surface-form variations. SCHEDBench provides a benchmark to test invariance to paraphrasing, revealing faithfulness issues and informing robust model development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9910dd16df5bc57ab08142d605f873c335b7e6267cc79a7d344e10615d4063f4"},"motivation":"This paper introduces SCHEDBench, a natural-language benchmark for evaluating combinatorial scheduling constraint faithfulness under surface-form variation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00991","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_masdrift_b9a896af","familyId":"bmf_1a3bba8a4a37","name":"MasDrift","oneLine":"Evaluates multi-agent systems on 600 productivity tasks measuring task completion and unauthorized action rate across different coordination architectures.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07556","pdf":"https://arxiv.org/pdf/2608.07556","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07556"},"evidence":{"snippet":"We introduce MasDrift, a benchmark of 600 benign productivity tasks across eight domains.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07556"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates multi-agent systems on 600 productivity tasks measuring task completion and unauthorized action rate across different coordination architectures.","whyItMatters":"Makes authorization preservation a measurable property of MAS design, exposing trade-offs between centralized and decentralized coordination for safe delegation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"b2296dd68558a7875bc749a9c2d60434f87a6d534cdd13b1835ce0be673e4547"},"motivation":"Multi-agent systems (MAS) decompose long-horizon tasks across supervisors and subagents, but delegated goals do not necessarily carry their original authorization boundaries.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07556","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MasDrift Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.07556","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_lethe_74a00c13","familyId":"bmf_7b3dcb32b39a","name":"Lethe","oneLine":"Lethe is a benchmark for federated unlearning in medical imaging. It evaluates twelve methods across eight task families, including classification, segmentation, denoising, cross-modality synthesis, and vision-language question answering. Three forgetting granularities (hospital, class, patient) are assessed against a retrained gold standard on utility, privacy, and cost.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.01094","pdf":"https://arxiv.org/pdf/2608.01094","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01094"},"evidence":{"snippet":"We present Lethe, a benchmark for federated unlearning in medical imaging.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01094"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Lethe is a benchmark for federated unlearning in medical imaging. It evaluates twelve methods across eight task families, including classification, segmentation, denoising, cross-modality synthesis, and vision-language question answering. Three forgetting granularities (hospital, class, patient) are assessed against a retrained gold standard on utility, privacy, and cost.","whyItMatters":"Existing unlearning evaluations primarily use natural images, leaving unclear whether methods transfer to clinical data. Lethe provides a shared protocol for medical imaging, enabling comparison of unlearning methods across diverse tasks and forgetting difficulties.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ba26a31b8f3950da5befde45ab5c250da19fa34eeafe20e7478f2d9ec8d7a70a"},"motivation":"Federated learning enables medical-imaging models to be trained across hospitals, and privacy law, most explicitly the GDPR ``right to be forgotten'', turns removing a hospital's, a class's, or a patient's influence from such a model into a federated unlearning problem.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.01094","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_hoprefusalbench_06f4ed80","familyId":"bmf_4c9fe07200c3","name":"HopRefusalBench","oneLine":"HopRefusalBench evaluates refusal behavior of search-augmented language model agents on multi-hop questions that are unanswerable, covering three causes of unanswerability and three chain topologies.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.01358","pdf":"https://arxiv.org/pdf/2608.01358","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01358"},"evidence":{"snippet":"We introduce HopRefusalBench, the first controlled benchmark of refusal within multi-hop search, comprising 889 unanswerable questions constructed from KILT-grounded entity paths.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01358"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HopRefusalBench evaluates refusal behavior of search-augmented language model agents on multi-hop questions that are unanswerable, covering three causes of unanswerability and three chain topologies.","whyItMatters":"Abstention benchmarks typically focus on single-hop queries, leaving evaluation gaps for failures that emerge during multi-hop reasoning and retrieval. This benchmark targets that gap and provides metrics for diagnosing refusal performance, aiding in improving agent reliability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"12bd6a9f71fd24469eed0ad9abf572d8dc30ced861d286d537f1681e08a5de36"},"motivation":"Search-augmented large language model agents are increasingly capable of solving knowledge-intensive tasks, but their behavior when a multi-hop question is fundamentally unanswerable remains poorly understood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.01358","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_dunphybench_f4ebbce5","familyId":"bmf_4789198ca55d","name":"DunphyBench","oneLine":"DunphyBench evaluates long-horizon embodied decision-making in housing environments, requiring agents to navigate and choose options aligned with multi-dimensional human preferences under partial observations.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Agents","Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.01456","pdf":"https://arxiv.org/pdf/2608.01456","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01456"},"evidence":{"snippet":"In this work, we propose DunphyBench, a new benchmark for evaluating agents on long-horizon human-centered embodied decision-making, where the agent must navigate through multiple embodied housing environments and make decisions that align with multi-dimensional human preferences.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01456"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DunphyBench evaluates long-horizon embodied decision-making in housing environments, requiring agents to navigate and choose options aligned with multi-dimensional human preferences under partial observations.","whyItMatters":"Addresses the gap in evaluating agents on long-horizon, human-centered decisions beyond procedural tasks, providing a reference for progress in integrating multimodal evidence and preference reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"52cbb7e90341a443df6a2c1eaf28568c944b39cf56792cab39a5f5d383eccc9e"},"motivation":"Agents are increasingly expected to act not only as task executors, but also as decision-makers on behalf of human users.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.01456","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_doc2ci_60d5a789","familyId":"bmf_8bd9fe5a8f8c","name":"Doc2CI","oneLine":"DOC2CI evaluates LLM-generated CI/CD configuration YAML against reference configurations from four CI services, measuring exact match and schema validity.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.01451","pdf":"https://arxiv.org/pdf/2608.01451","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.01451"},"evidence":{"snippet":"We introduce DOC2CI, a benchmark of 3,363 description-to-YAML pairs collected from the official documentation of four CI services, and evaluate 14 open-weight models from 7B-34B parameters together with GPT-4o and GPT-4.1, producing over 53,000 configurations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.01451"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DOC2CI evaluates LLM-generated CI/CD configuration YAML against reference configurations from four CI services, measuring exact match and schema validity.","whyItMatters":"It quantifies the gap between LLM-generated configuration validity and reference similarity, highlighting the need for schema-aware evaluation in configuration generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a0c3a19d7cff8e576878c0f419ebb72da9a1b1cd6be61ae444f8907390adc617"},"motivation":"Adopting Continuous Integration (CI) often requires writing YAML configurations that are error-prone and challenging to maintain.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.01451","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_boundary-bench_5415f6d9","familyId":"bmf_29b2f52f6f51","name":"Boundary-Bench","oneLine":"Boundary-Bench is an open-source plugin that adds configurable security policy levels to Terminal-Bench, enabling evaluation of coding agents under constraints like scoped credentials, restricted egress, read-only filesystems, and non-root execution.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CR"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-08-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02670","pdf":"https://arxiv.org/pdf/2608.02670","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.02670"},"evidence":{"snippet":"We release Boundary-Bench, an open-source hardening plugin enabling policy-constrained evaluation of coding agents on Terminal-Bench and compatible benchmarks.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02670"},"ranking":{"30d":{"score":49,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Boundary-Bench is an open-source plugin that adds configurable security policy levels to Terminal-Bench, enabling evaluation of coding agents under constraints like scoped credentials, restricted egress, read-only filesystems, and non-root execution.","whyItMatters":"Existing coding agent benchmarks assume permissive sandboxes, leaving a gap in understanding performance under real-world security policies. This benchmark provides a standardized way to measure success and efficiency trade-offs across policy levels, informing model selection for deployment in hardened environments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9a2d68a81eda969254befd1930f2ad4b0a24d9426358d5cb66bb11f832e4ce48"},"motivation":"Coding agents increasingly run inside organizations whose security controls (scoped credentials, restricted egress, read-only filesystems, non-root execution) constrain them like any other software.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02670","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Boundary-Bench authors","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.02670","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_vul4py_70948b69","familyId":"bmf_f19614b79970","name":"Vul4Py","oneLine":"Vul4Py evaluates automated vulnerability repair in Python across 100 real vulnerabilities from 60 open-source projects, with paired exploit and functional oracles.","area":"Code & Software","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00692","pdf":"https://arxiv.org/pdf/2608.00692","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00692"},"evidence":{"snippet":"We present Vul4Py, a Python AVR benchmark in which every entry carries a paired oracle: an exploit oracle that must fail on the vulnerable revision and pass on the fixed one, together with a project-native pytest functional oracle that must pass on both.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00692"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Vul4Py evaluates automated vulnerability repair in Python across 100 real vulnerabilities from 60 open-source projects, with paired exploit and functional oracles.","whyItMatters":"It addresses the gap of missing functional regression checks in existing Python AVR benchmarks, providing a more reliable comparison of repair methods.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f35fb45d613ec02aec2c8635a49dd69e31c95fdcf1c2def8036527a936117c62"},"motivation":"Automated Vulnerability Repair (AVR) has advanced rapidly across program analysis, machine learning, and Large Language Models (LLMs), but a verifiable, head-to-head comparison of AVR approaches on Python is still missing.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00692","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_treeprobe_2c533886","familyId":"bmf_d1c4a8ee6c7a","name":"TreeProbe","oneLine":"TreeProbe is a dataset of 4,719 expert-adjudicated items covering 467 diseases and 10 subtasks across the Tibetan Tree of Medicine framework, used to evaluate cultural bias in LLMs.","area":"Safety & Trustworthiness","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Software & Cloud","Pharma & Biotech"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-08-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00640","pdf":"https://arxiv.org/pdf/2608.00640","project":"https://anonymous.4open.science/r/TreeProbe/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00640"},"evidence":{"snippet":"To address this gap, we introduce TreeProbe, the first cultural-bias benchmark organized around the native Tree of Medicine framework in Tibetan medicine.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00640"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TreeProbe is a dataset of 4,719 expert-adjudicated items covering 467 diseases and 10 subtasks across the Tibetan Tree of Medicine framework, used to evaluate cultural bias in LLMs.","whyItMatters":"It addresses the lack of quantitative tools for assessing cultural bias in Tibetan medicine, offering a diagnostic lens for epistemic fairness in medical AI, though it primarily supports the paper's analysis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"42020c58ef3c04408bca30bc12b2681995ac9a59f92675e4bdfd5e04b0790971"},"motivation":"Large language models are increasingly viewed as a potential means of mitigating global health inequities, yet their outputs often reflect dominant high-resource medical traditions and provide limited coverage of traditional medical knowledge systems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00640","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness","Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_s2m-bench_0da86db5","familyId":"bmf_9b8a17f82d4d","name":"S2M-Bench","oneLine":"S2M-Bench evaluates the task of reconstructing mind maps from lecture slides, comprising 12,774 slide pages from 24 university courses with expert-annotated mind maps. The evaluation framework integrates ground-truth comparison, structure conformity analysis, and VLM-as-a-Judge scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00610","pdf":"https://arxiv.org/pdf/2608.00610","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00610"},"evidence":{"snippet":"For systematic evaluation, we introduce S2M-Bench, a benchmark comprising 12,774 slide pages with expert-annotated mind maps spanning 24 university courses.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00610"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"S2M-Bench evaluates the task of reconstructing mind maps from lecture slides, comprising 12,774 slide pages from 24 university courses with expert-annotated mind maps. The evaluation framework integrates ground-truth comparison, structure conformity analysis, and VLM-as-a-Judge scoring.","whyItMatters":"Existing benchmarks do not address automatic generation and evaluation of mind maps from educational slides, a task requiring balance of local and global knowledge. S2M-Bench provides a systematic evaluation framework to advance intelligent education tools by enabling comparison of models on this task.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"91cfac840ab0ae0681fc5ea2f9c1950c0568e6755e31d5bf72f999511b32587b"},"motivation":"Generating mind maps from lecture slides can help learners efficiently assimilate fragmented knowledge, promising substantial benefits for intelligent education.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00610","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_findeepindicator_6effb730","familyId":"bmf_5d885cb8e46f","name":"FinDeepIndicator","oneLine":"FinDeepIndicator evaluates deep research agents on end-to-end financial indicator construction across four stages: formula specification, data collection, indicator calculation, and answer generation. It includes 3,350 QA pairs from U.S. and Chinese markets, 10 years of historical data, and 800 listed companies.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00764","pdf":"https://arxiv.org/pdf/2608.00764","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00764"},"evidence":{"snippet":"In this work, we propose FinDeepIndicator, the first benchmark dedicated to evaluating Deep Research (DR) agents in end-to-end financial indicator construction.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00764"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"FinDeepIndicator evaluates deep research agents on end-to-end financial indicator construction across four stages: formula specification, data collection, indicator calculation, and answer generation. It includes 3,350 QA pairs from U.S. and Chinese markets, 10 years of historical data, and 800 listed companies.","whyItMatters":"Existing financial benchmarks focus on answer accuracy and assume data is provided, leaving the intermediate process of indicator construction unassessed. FinDeepIndicator bridges this gap by evaluating the full process, providing insights for building more capable and trustworthy agents in finance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4c61a3efc086d6cfbce3796c35c54e47b496117060935a889fca74fe4515706c"},"motivation":"Financial indicators are essential tools for transforming raw financial data into interpretable measures for various downstream tasks, such as valuation, risk assessment, and economic analysis.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00764","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_edupluginbench_5240df80","familyId":"bmf_74fa30a0270a","name":"EduPluginBench","oneLine":"EduPluginBench is an executable benchmark for evaluating code-generation models on producing plugins that meet governed ecosystem requirements including least privilege, telemetry consent, provenance, and bounded failure. It uses 1,440 mutants and 120 clean references with staged admission levels P0-P4.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-08-01","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2608.00739","pdf":"https://arxiv.org/pdf/2608.00739","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00739"},"evidence":{"snippet":"We introduce EduPluginBench, an executable benchmark and staged admission method for generated plugins in governed software ecosystems.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00739"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EduPluginBench is an executable benchmark for evaluating code-generation models on producing plugins that meet governed ecosystem requirements including least privilege, telemetry consent, provenance, and bounded failure. It uses 1,440 mutants and 120 clean references with staged admission levels P0-P4.","whyItMatters":"Compilation and functional tests do not ensure compliance with security and governance constraints. EduPluginBench provides a staged admission method targeting these gaps, enabling assessment of generated plugins in governed environments, which is critical for safe deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2932025763fd783563ed3470cceefc30734d2540086e0a2ef320354e6012e47e"},"motivation":"Code-generation models can produce executable components, but compilation and functional tests do not establish compliance with least privilege, telemetry consent, provenance, privileged-write authority, lifecycle constraints, or bounded failure.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00739","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_advplan-bench_7a0b9910","familyId":"bmf_9f699b02a07d","name":"AdvPlan-Bench","oneLine":"AdvPlan-Bench is an offline benchmark for adversarial evaluation of structured plan-generation agents. It uses typed action chains, adversarial response sets, selector diagnostics, and metrics like BLUE-vs-RED advantage and Nash-gap. Includes 150 synthetic scenarios across five planning templates.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-08-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00832","pdf":"https://arxiv.org/pdf/2608.00832","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00832"},"evidence":{"snippet":"We introduce AdvPlan-Bench, an offline benchmark for adversarial evaluation of structured plan-generation agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00832"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AdvPlan-Bench is an offline benchmark for adversarial evaluation of structured plan-generation agents. It uses typed action chains, adversarial response sets, selector diagnostics, and metrics like BLUE-vs-RED advantage and Nash-gap. Includes 150 synthetic scenarios across five planning templates.","whyItMatters":"Plan quality is often evaluated in isolation, but realistic tasks require considering adversarial responses. AdvPlan-Bench provides a reproducible method to study adversarial plan evaluation, response-budget sensitivity, and candidate frontiers, informing robust planning agent design.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ebfe1dc5a44777a398834bb733b3accb0e333e6639d5aca3978b5b5e7a682117"},"motivation":"Structured plan-generation agents are often evaluated as if a plan has quality in isolation, yet many realistic planning tasks require asking how a candidate behaves when another agent can search for responses.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00832","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_scifigplag-bench_b0402713","familyId":"bmf_98ef94f5375b","name":"SciFigPlag-Bench","oneLine":"SciFigPlag-Bench evaluates provenance-aware reasoning for scientific figure plagiarism detection. It includes 2,582 positive and 2,541 negative pairs, with tasks for pairwise detection, source attribution, reuse-type classification, and reuse correspondence localization.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.29124","pdf":"https://arxiv.org/pdf/2607.29124","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.29124"},"evidence":{"snippet":"We present SciFigPlag-Bench, a benchmark for provenance-aware reasoning over scientific figures in scholarly documents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.29124"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SciFigPlag-Bench evaluates provenance-aware reasoning for scientific figure plagiarism detection. It includes 2,582 positive and 2,541 negative pairs, with tasks for pairwise detection, source attribution, reuse-type classification, and reuse correspondence localization.","whyItMatters":"Figure plagiarism is underexplored and general similarity benchmarks do not assess provenance. SciFigPlag-Bench provides a factorized taxonomy and diagnostic tasks to evaluate multimodal models on fine-grained provenance reasoning, aiding research integrity.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0bf388a17f76af0d49accf5c30efef19253cb91f2bd019d78ea06acc39adc1e7"},"motivation":"Scientific figures often encode the visual evidence behind scientific findings, yet figure plagiarism remains underexplored as a benchmarked multimodal evaluation problem.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.29124","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mmshopbench_a4cdc83b","familyId":"bmf_b4f7e925e936","name":"MMShopBench","oneLine":"MMShopBench is a real-log benchmark for multimodal multi-turn shopping agents. It uses cleaned shopping logs with annotations for purchase intent and mandatory product requirements. Agents must infer requirements from images and dialogue, retrieve candidates, and verify product satisfaction.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.29002","pdf":"https://arxiv.org/pdf/2607.29002","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.29002"},"evidence":{"snippet":"We introduce MMShopBench, the first real-log benchmark for multimodal, multi-turn shopping agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.29002"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MMShopBench is a real-log benchmark for multimodal multi-turn shopping agents. It uses cleaned shopping logs with annotations for purchase intent and mandatory product requirements. Agents must infer requirements from images and dialogue, retrieve candidates, and verify product satisfaction.","whyItMatters":"Existing benchmarks often rely on text-only or synthetic requests, missing complex real-world multimodal shopping needs. MMShopBench provides a realistic evaluation to advance shopping agents, with an offline sandbox for reproducible experimentation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d1a96fc9628e62a6a2cedc0d6b03def4a0b9e8b0e9447f8634fabfecbce69001"},"motivation":"Online shoppers increasingly turn to AI shopping assistants, using images and multi-turn dialogue to express and refine product needs that are difficult to articulate in text alone.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.29002","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mirrorcraft_4e4ff3e9","familyId":"bmf_17f3ccda834b","name":"MirrorCraft","oneLine":"MirrorCraft is a paired benchmark for evaluating LLM-based agents under hidden rule changes in Minecraft. Each Mirror world copies a Vanilla world with modified server-side rules, while terrain and objectives match. It includes five biomes, six rule suites, three objectives, and uses deterministic milestones and Rule Intervention Effect.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.29218","pdf":"https://arxiv.org/pdf/2607.29218","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.29218"},"evidence":{"snippet":"In this paper, we introduce MirrorCraft, a paired benchmark for evaluating agents under hidden rule changes in Minecraft.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.29218"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MirrorCraft is a paired benchmark for evaluating LLM-based agents under hidden rule changes in Minecraft. Each Mirror world copies a Vanilla world with modified server-side rules, while terrain and objectives match. It includes five biomes, six rule suites, three objectives, and uses deterministic milestones and Rule Intervention Effect.","whyItMatters":"Most Minecraft benchmarks use fixed mechanics, not testing adaptability to rule changes. MirrorCraft provides a controlled setting to study agent performance under hidden rule modifications, important for developing agents that can handle dynamic environments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"047b7389d34807385728a55dbff7e5866c34b714d09e80dc54496f6f82f01002"},"motivation":"With the prosperity of the large language models (LLMs), it has become an interesting topic: how do LLM-based agents work in Minecraft?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.29218","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_metaroute-bench_a64b1073","familyId":"bmf_fd31642f325d","name":"MetaRoute-Bench","oneLine":"MetaRoute-Bench is an open framework for comparing meta-decision policies in agentic workflow routing. Contains 180 synthetic task profiles, eight routing policies, and 30 seeds, evaluating success, cost, and latency through 43,200 traces. Metrics include success rate and paired confidence intervals.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00107","pdf":"https://arxiv.org/pdf/2608.00107","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00107"},"evidence":{"snippet":"We present MetaRoute-Bench, an open, inspectable framework for comparing meta-decision policies under a shared execution model.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00107"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MetaRoute-Bench is an open framework for comparing meta-decision policies in agentic workflow routing. Contains 180 synthetic task profiles, eight routing policies, and 30 seeds, evaluating success, cost, and latency through 43,200 traces. Metrics include success rate and paired confidence intervals.","whyItMatters":"Meta-decisions in agentic systems are often embedded and evaluated only via aggregate task accuracy. MetaRoute-Bench provides a shared execution model to compare routing policies, enabling analysis of tradeoffs between success, cost, and latency.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b26f1040e6a5316c1e7822e89434fcac4045ac80b2665ade155a93efcd9bedd2"},"motivation":"Agentic systems must repeatedly decide whether to answer directly, decompose a task, invoke a tool, execute code, delegate to a specialist, verify an intermediate result, or recover from failure.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00107","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_mdarena_a3bdd691","familyId":"bmf_7ef0c7c56d53","name":"MDArena","oneLine":"MDArena evaluates coding agents on 50 containerized molecular dynamics tasks from active research projects, covering trajectory analysis, system preparation, free-energy protocols, and enhanced sampling, with strict pass-based scoring.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["physics.chem-ph"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02642","pdf":"https://arxiv.org/pdf/2608.02642","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.02642"},"evidence":{"snippet":"To address this issue, we introduce MDArena, a benchmark of 50 containerized tasks drawn from active biomolecular simulation projects, spanning 29 molecular systems and 14 broad research protocols, including trajectory analysis, complex system preparation, free-energy protocols, and enhanced sampling.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02642"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MDArena evaluates coding agents on 50 containerized molecular dynamics tasks from active research projects, covering trajectory analysis, system preparation, free-energy protocols, and enhanced sampling, with strict pass-based scoring.","whyItMatters":"Bridges the gap between supervised AI assistance and autonomous research reliability by providing a reproducible platform for tracking progress in automating complex scientific workflows.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"63b89ecbcbac162af4010b27a3e19f290c35d34aff1eae74b897e5d271103f0c"},"motivation":"Accelerating scientific discovery is among the most consequential applications of AI, and computational biomolecular simulation stands out as a particularly promising target within this broader effort.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02642","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_m3-duplexbench_afd47e79","familyId":"bmf_8b667f9a2394","name":"M3-DuplexBench","oneLine":"M3-DuplexBench evaluates full-duplex spoken dialogue models in multi-turn, multilingual (English and Japanese), multidomain settings, with multiple dialogue context settings and turn-taking analysis.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.29125","pdf":"https://arxiv.org/pdf/2607.29125","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.29125"},"evidence":{"snippet":"We propose M3-DuplexBench, a multi-turn, multilingual, multidomain benchmark for FDSDSs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.29125"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"M3-DuplexBench evaluates full-duplex spoken dialogue models in multi-turn, multilingual (English and Japanese), multidomain settings, with multiple dialogue context settings and turn-taking analysis.","whyItMatters":"Addresses the lack of fair multi-turn comparisons in full-duplex dialogue systems, enabling analysis across languages, domains, and context settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"db782de0dec032683a528a2956e516d9940a7a9d3ac98815983d456e9789d0d4"},"motivation":"Full-duplex spoken dialogue systems (FDSDSs) can listen while speaking, enabling natural behaviors such as smooth turn-taking, backchannel handling, and user barge-in handling.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.29125","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_loopsbench_86719709","familyId":"bmf_0c0fee9fe39e","name":"LoopsBench","oneLine":"LOOPSBENCH evaluates coding agents on long-horizon tasks structured as dependency DAGs with flow-aware test release and regression obligations, spanning 8 languages and 9 domains.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00267","pdf":"https://arxiv.org/pdf/2608.00267","project":"https://loopsbench.ai/","code":"https://github.com/microsoft/Loopsbench","data":null,"hfPaper":"https://huggingface.co/papers/2608.00267"},"evidence":{"snippet":"We introduce LOOPSBENCH, a long-horizon benchmark for loop engineering in coding agent evaluation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-reviewed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":null,"githubStars":21,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00267"},"ranking":{"30d":{"score":71,"rank":30,"coverage":0.85,"confidence":"High"},"90d":{"score":65,"rank":102,"coverage":0.7,"confidence":"Medium"}},"description":"LOOPSBENCH evaluates coding agents on long-horizon tasks structured as dependency DAGs with flow-aware test release and regression obligations, spanning 8 languages and 9 domains.","whyItMatters":"Provides a benchmark for loop engineering in sustained software development, assessing planning, implementation, and recovery over long horizons.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"05fe1d8410d59b6afe5cdae1ed2a975ac29afc278a5a0b431692ebd9672933aa"},"motivation":"Coding agent infrastructure is shifting from harness engineering toward loop engineering as coding agents are deployed for sustained long-horizon software development.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"source-reviewed","reviewedAt":"2026-08-19","sources":["https://github.com/microsoft/Loopsbench","https://loopsbench.ai/","https://arxiv.org/abs/2608.00267"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00267","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"releaseDates":{"firstPublicAt":"2026-07-03","paperV1At":"2026-07-31"},"publishers":[{"name":"Microsoft","organizationType":"company-research-lab","sourceUrl":"https://github.com/microsoft/Loopsbench","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_livecodebench_fa145f1d","familyId":"bmf_b33aa0adf0e0","name":"LiveCodeBench","oneLine":"CurveShift analyzes progress on LiveCodeBench, releasing a difficulty panel of 66 models and 1,055 problems.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00355","pdf":"https://arxiv.org/pdf/2608.00355","project":null,"code":"https://github.com/harvenstar/CurveShift","data":null,"hfPaper":"https://huggingface.co/papers/2608.00355"},"evidence":{"snippet":"We release the LiveCodeBench Difficulty Panel (66 dated models x 1,055 problems) and our analysis code.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00355"},"ranking":{"30d":{"score":50,"rank":74,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":283,"coverage":0.55,"confidence":"Low"}},"description":"CurveShift analyzes progress on LiveCodeBench, releasing a difficulty panel of 66 models and 1,055 problems.","whyItMatters":"Provides insights into whether progress is scalar or shaped by task difficulty, but does not define a new evaluation benchmark.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ae80d13489ec097ddf97a43759b4f16ca26250966a72f9a4e69162f9ad4704d9"},"motivation":"Progress in large language models is often summarized using a single scalar measure, such as a time horizon, a latent ability estimate, or an aggregate benchmark score.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00355","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_infrabench_aa520bfd","familyId":"bmf_d9f6fa305b39","name":"InfraBench","oneLine":"InfraBench evaluates AI agents on realistic infrastructure tasks across the system stack and lifecycle, with fine-grained per-check risk assessment and a public leaderboard.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.11234","pdf":"https://arxiv.org/pdf/2608.11234","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.11234"},"evidence":{"snippet":"We present InfraBench, a benchmark suite for evaluating AI agents on realistic infrastructure tasks across the full system stack and full operational lifecycle with fine-grained risk assessment.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.11234"},"ranking":{"30d":{"score":47,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"InfraBench evaluates AI agents on realistic infrastructure tasks across the system stack and lifecycle, with fine-grained per-check risk assessment and a public leaderboard.","whyItMatters":"Addresses the complexity of infrastructure management by providing a benchmark for assessing agent performance across risk dimensions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"aeed5bfd74ed096059e6b63aa447c3d34d752c0caab46918b57d07ccd2d5a67b"},"motivation":"Managing modern computing infrastructure has become a steadily harder problem due to the ever-increasing complexity.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.11234","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_inferq_453f0fc4","familyId":"bmf_429a10ce02b2","name":"InferQ","oneLine":"InferQ is a database-oriented benchmark for quantum circuit simulation, generating compositional circuits and emitting SQL workloads with feature extraction for workload characterization.","area":"Language & Knowledge","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Quantum Technology"],"capabilities":[],"topics":["quant-ph"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.29134","pdf":"https://arxiv.org/pdf/2607.29134","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.29134"},"evidence":{"snippet":"We present InferQ, a database-oriented benchmark for quantum circuit simulation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.29134"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"InferQ is a database-oriented benchmark for quantum circuit simulation, generating compositional circuits and emitting SQL workloads with feature extraction for workload characterization.","whyItMatters":"Enables systematic database research on quantum simulation, supporting query optimization, physical design, and engine-level evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dd131eb241e99fb079763dc6679ad54b5390423fabcf40411c15a5b543e7aafc"},"motivation":"Recent work suggests that relational database management systems (RDBMSs) can execute quantum circuit simulation by compiling the simulation into SQL workloads (primarily join-and-aggregate tensor contractions).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"presentation at ACM SIGMOD 2027 and publication in the Proceedings of the ACM on Management of Data (PACMMOD)","evidence":"Accepted for presentation at ACM SIGMOD 2027 and publication in the Proceedings of the ACM on Management of Data (PACMMOD). This arXiv version is an extended technical report that includes the complete appendix","evidenceUrl":"https://arxiv.org/abs/2607.29134","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"presentation at ACM SIGMOD 2027 and publication in the Proceedings of the ACM on Management of Data (PACMMOD)","reviewStatus":"accepted","decisionRaw":"Accepted for presentation at ACM SIGMOD 2027 and publication in the Proceedings of the ACM on Management of Data (PACMMOD). This arXiv version is an extended technical report that includes the complete appendix","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.29134","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted for presentation at ACM SIGMOD 2027 and publication in the Proceedings of the ACM on Management of Data (PACMMOD). This arXiv version is an extended technical report that includes the complete appendix","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_hy-multiturn_edce774f","familyId":"bmf_394a3bb3e96c","name":"Hy-MultiTurn","oneLine":"Hy-MultiTurn is a Chinese benchmark for deep multi-turn dialogue understanding, with six controlled evaluation modes covering constraint memory, object localization, and action suppression across 12-76 turn dialogues.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.29196","pdf":"https://arxiv.org/pdf/2607.29196","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.29196"},"evidence":{"snippet":"To address these limitations, we analyze real chatbot failures to identify six recurring mechanisms and use them to define six controlled evaluation modes in Hy-MultiTurn, a Chinese benchmark for deep multi-turn dialogue understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.29196"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Hy-MultiTurn is a Chinese benchmark for deep multi-turn dialogue understanding, with six controlled evaluation modes covering constraint memory, object localization, and action suppression across 12-76 turn dialogues.","whyItMatters":"Evaluates long multi-turn dialogue capabilities that existing benchmarks miss, offering controlled modes for diagnosing model failures.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c1fbf03fe6a286a621e4a0fd9065deff22a6f54896d0d6c9569b0f9d733c6444"},"motivation":"Long-running multi-turn interactions with chatbots and agents are now common, and a correct response often depends on remembering earlier details, tracking later revisions, identifying intended objects or referents, and withholding action when required conditions are unmet.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.29196","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_friendbench_e9086fd3","familyId":"bmf_ee7ae5c537f8","name":"FriendBench","oneLine":"FriendBench evaluates dyadic familiarity inference in humans and models using 96 dyads, comparing accuracy across modalities.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.29602","pdf":"https://arxiv.org/pdf/2607.29602","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.29602"},"evidence":{"snippet":"We introduce FriendBench, a benchmark for inferring whether two people are already familiar or are meeting as strangers, from a 20-second clip of a dyadic ice-breaker conversation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.29602"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FriendBench evaluates dyadic familiarity inference in humans and models using 96 dyads, comparing accuracy across modalities.","whyItMatters":"Provides insight into social perception differences, but does not offer a standardized comparison path for models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"49ae30534c2f22ded8a5e87ad723e2bc4e41fd6cbe1291869577a97c0f11ee9c"},"motivation":"Reading a social situation often depends on behavior, not words alone.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.29602","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_extractbench_2ecdf546","familyId":"bmf_b7c261db99d3","name":"ExtractBench","oneLine":"ExtractBench evaluates schema-guided extraction from enterprise documents. Given a document and a user-defined JSON schema, systems must return schema-valid JSON with correct values, include every record of repeated structures, mark missing fields as null, and provide source evidence. The benchmark includes 4,869 pages across 370 documents, 8 business domains, and 67 document types, with tags for challenge, perception, table structure, length, and domain. Scoring uses unified value F1 for value accuracy and word- and page-level F1 for grounding.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.29677","pdf":"https://arxiv.org/pdf/2607.29677","project":null,"code":"https://github.com/run-llama/ExtractBench","data":"https://huggingface.co/datasets/llamaindex/ExtractBench","hfPaper":"https://huggingface.co/papers/2607.29677"},"evidence":{"snippet":"We present ExtractBench, a benchmark for schema-guided extraction and, to our knowledge, the first to score value accuracy, record completeness at scale, grounding, and measured cost together.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":24,"hfDailySubmittedAt":"2026-08-03T00:00:00.000Z","githubStars":62,"githubScope":"benchmark_repo","hfDatasetDownloads":7457,"hfDatasetLikes":21},"source":{"type":"arxiv","id":"2607.29677"},"ranking":{"30d":{"score":93,"rank":2,"coverage":1.0,"confidence":"High","datasetDownloadRank":1,"datasetRankPopulation":20},"90d":{"score":92,"rank":4,"coverage":1.0,"confidence":"High","datasetDownloadRank":2,"datasetRankPopulation":51}},"description":"ExtractBench evaluates schema-guided extraction from enterprise documents. Given a document and a user-defined JSON schema, systems must return schema-valid JSON with correct values, include every record of repeated structures, mark missing fields as null, and provide source evidence. The benchmark includes 4,869 pages across 370 documents, 8 business domains, and 67 document types, with tags for challenge, perception, table structure, length, and domain. Scoring uses unified value F1 for value accuracy and word- and page-level F1 for grounding.","whyItMatters":"Enterprise workflows increasingly rely on agents for schema-guided extraction, where errors can lead to wrong payments or decisions. ExtractBench addresses the lack of a benchmark that jointly measures value accuracy, completeness, grounding, and cost, providing a standardized evaluation for comparing extraction systems on realistic document types and lengths.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"89d3015ed6801258917c496c948fb9c04f09e17fbe7b0257ca0d8f949129f8fe"},"motivation":"Enterprise workflows increasingly rely on agents for \\emph{schema-guided extraction}: given a document and a user-defined schema, the agent faithfully follows the schema to produce the correct output with source evidence as grounding metadata.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.29677","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LlamaIndex","organizationType":"company-research-lab","sourceUrl":"https://github.com/run-llama/ExtractBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_dungeonbench_a497b825","familyId":"bmf_d20232b4c917","name":"DungeonBench","oneLine":"DungeonBench evaluates tactical reasoning in Dungeons & Dragons combat with two tracks: Encounter for single fights and Day for linked encounters with persistent resources, using a shared decision stream of complete tactical observations and legal options.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.29577","pdf":"https://arxiv.org/pdf/2607.29577","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.29577"},"evidence":{"snippet":"We introduce DungeonBench, a benchmark for tactical reasoning in Dungeons & Dragons combat, built to cover the vast majority of combat-relevant 2014 System Reference Document content whose effects can be resolved by the simulator while retaining mechanics that simplified combat simulators often abstract away.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.29577"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DungeonBench evaluates tactical reasoning in Dungeons & Dragons combat with two tracks: Encounter for single fights and Day for linked encounters with persistent resources, using a shared decision stream of complete tactical observations and legal options.","whyItMatters":"Current benchmarks often under-test rules-rich tactical reasoning where geometry, timing, resources, and rule interactions matter. DungeonBench fills this gap by providing a reproducible simulator-based environment with clear scoring, allowing comparison of policies from heuristic controllers to language models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f2f2c45dc95d1fedc1b91d2dfe45ef30f1361711c32bbf8a32f74cad93d009ca"},"motivation":"Games and simulators make valuable benchmarks by turning decisions into measurable outcomes, but many current suites under-test rules-rich tactical reasoning: the ability to choose well when geometry, timing, resources, objectives, and rule interactions all matter at once.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.29577","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DungeonBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.29577","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_agenthpobench_61915136","familyId":"bmf_c788cbfe6bdb","name":"AgentHPOBench","oneLine":"AgentHPOBench evaluates LLM agents as sequential hyperparameter optimizers across 30 executable ML tasks. Agents observe accumulated configurations, metrics, and logs, then propose the next configuration. Scoring compares agents and conventional HPO baselines under a unified protocol.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.29626","pdf":"https://arxiv.org/pdf/2607.29626","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.29626"},"evidence":{"snippet":"To address this gap, we introduce AgentHPOBench, a sequential benchmark comprising 30 executable machine learning tasks across seven research categories.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.29626"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AgentHPOBench evaluates LLM agents as sequential hyperparameter optimizers across 30 executable ML tasks. Agents observe accumulated configurations, metrics, and logs, then propose the next configuration. Scoring compares agents and conventional HPO baselines under a unified protocol.","whyItMatters":"Existing benchmarks do not directly assess whether agents can interpret experimental evidence and use it to guide subsequent hyperparameter decisions. AgentHPOBench provides a repeatable protocol for measuring iterative decision-making, filling a gap in evaluating autonomous scientific agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6e96b78ef967b4aaa652baba8aedfc43c3141d44ed9ae91a41f1399a880dedae"},"motivation":"As LLMs evolve from code completion systems into autonomous scientific agents, evaluating their ability to conduct experiments has become increasingly important.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.29626","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AgentHPOBench team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.29626","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_repbench_7ff35873","familyId":"bmf_b37ff0b1a335","name":"RepBench","oneLine":"RepBench compiles 353 public benchmarks into 46,149 probe texts spanning 94 capabilities, with a taxonomy of 182 clusters in 13 families. It evaluates representation readout methods across 12 models under cross-benchmark transfer, providing a reusable closed-loop pipeline for capability-aligned probing.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.28008","pdf":"https://arxiv.org/pdf/2607.28008","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.28008"},"evidence":{"snippet":"We present RepBench, a benchmark-grounded data layer for capability-aligned representation probing.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.28008"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RepBench compiles 353 public benchmarks into 46,149 probe texts spanning 94 capabilities, with a taxonomy of 182 clusters in 13 families. It evaluates representation readout methods across 12 models under cross-benchmark transfer, providing a reusable closed-loop pipeline for capability-aligned probing.","whyItMatters":"RepBench addresses the lack of comparable and reproducible evaluation for representation engineering by grounding probes in multiple public benchmarks, reducing single-source bias and enabling meaningful comparison of readout methods and aggregation criteria across models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d8bab187c774e445ecd97f0923d704197d7c5ce7afbffd994fc236376409ea4a"},"motivation":"Representation engineering reads and steers capability directions in large language models, yet methods are typically evaluated on paper-specific synthetic data.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.28008","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_pathview-bench_295675e0","familyId":"bmf_8ae3ed6f0412","name":"PathView-Bench","oneLine":"PathVU is a benchmark for fine-grained multiscale visual understanding in pathology, built from 23 public datasets with human-supervised labels and spatial annotations, covering region and slide fields of view, with 14 VQA-style tasks and 308k samples.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.28318","pdf":"https://arxiv.org/pdf/2607.28318","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.28318"},"evidence":{"snippet":"We introduce PathVU, a vision-anchored benchmark for fine-grained and multiscale visual understanding in computational pathology.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.28318"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PathVU is a benchmark for fine-grained multiscale visual understanding in pathology, built from 23 public datasets with human-supervised labels and spatial annotations, covering region and slide fields of view, with 14 VQA-style tasks and 308k samples.","whyItMatters":"PathVU aims to evaluate fine-grained visual understanding in pathology MLLMs, potentially offering more detailed assessment than existing benchmarks that focus on final diagnoses.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3e923a1c3ccc600627a0c4f8447f237a4766347eddb9d236a45e9e93bbab165e"},"motivation":"Multimodal large language models (MLLMs) are increasingly used to analyze pathology images.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.28318","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_orca-bench_79a9e07b","familyId":"bmf_ee99b02dd1db","name":"ORCA-bench","oneLine":"ORCA-bench evaluates language model agents on root cause analysis in a production-fidelity oncall setting, using a live OpenTelemetry-instrumented microservice system with 1,079 tasks varying in report specificity, time-to-detection, and fault co-occurrence. Agents access metrics, logs, traces, and source code via real telemetry interfaces.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.28545","pdf":"https://arxiv.org/pdf/2607.28545","project":"https://hub.harborframework.com/datasets/orca-bench/orca-bench","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.28545"},"evidence":{"snippet":"We introduce ORCA-bench, a benchmark that puts general-purpose coding agents in a production-fidelity oncall setting.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.28545"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ORCA-bench evaluates language model agents on root cause analysis in a production-fidelity oncall setting, using a live OpenTelemetry-instrumented microservice system with 1,079 tasks varying in report specificity, time-to-detection, and fault co-occurrence. Agents access metrics, logs, traces, and source code via real telemetry interfaces.","whyItMatters":"Standard coding benchmarks do not capture the complexity of oncall RCA, where agents must reason over noisy, heterogeneous data. ORCA-bench provides a reproducible testbed to measure agent readiness for production reliability tasks, revealing a significant performance gap even for frontier models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cd8c69fe2003f3b637d6ae1908529e54338a40d5249f2b3e299e0934d6c49976"},"motivation":"Large language models can write, patch, and search code, but oncall root cause analysis (RCA) demands something different: reasoning over noisy metrics, logs, traces, and source code, starting from ambiguous user-facing reports, often hours after the incident began.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.28545","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Harbor Framework","organizationType":"benchmark-organization","sourceUrl":"https://hub.harborframework.com/datasets/orca-bench/orca-bench","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_openscitoolbench_fd41e9ac","familyId":"bmf_867272727e97","name":"OpenSciToolBench","oneLine":"OpenSciToolBench is a benchmark with 900 tasks across four difficulty levels for evaluating LLM agents in open-world scientific tool acquisition.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.28692","pdf":"https://arxiv.org/pdf/2607.28692","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.28692"},"evidence":{"snippet":"Moreover, we introduce OpenSciToolBench, a benchmark containing 900 realistic tasks across four difficulty levels.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.28692"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OpenSciToolBench is a benchmark with 900 tasks across four difficulty levels for evaluating LLM agents in open-world scientific tool acquisition.","whyItMatters":"The benchmark supports a specific agent system's evaluation and lacks a standalone comparison path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ba5b334155f3e56e65c4ac59f900cbe447132860cd3112ae89e524b6c5a29cc7"},"motivation":"Large language model (LLM) agents have been increasingly adopted in scientific research for organizing and invoking specialized computational tools.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.28692","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_mpie-bench_16361495","familyId":"bmf_e29db8d408ba","name":"MPIE-Bench","oneLine":"MPIE-Bench evaluates multi-person image editing models on tasks involving multiple named people in contact interactions such as embrace, carry, or grapple. The benchmark provides a 2,500-sample test set with 14 interaction categories and four contact densities, and scores outputs on six axes including anatomy and interaction via mesh reconstruction.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27616","pdf":"https://arxiv.org/pdf/2607.27616","project":null,"code":"https://github.com/AnnLin0628/mpie-bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.27616"},"evidence":{"snippet":"We introduce MPIE-Bench, a 2,500-sample benchmark of video-mined editing triplets spanning 405 scenes, 14 interaction categories, and four contact densities (C0-C3).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":39,"hfDailySubmittedAt":"2026-07-31T00:00:00.000Z","githubStars":7,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27616"},"ranking":{"30d":{"score":78,"rank":13,"coverage":0.85,"confidence":"High"},"90d":{"score":61,"rank":126,"coverage":0.7,"confidence":"Medium"}},"description":"MPIE-Bench evaluates multi-person image editing models on tasks involving multiple named people in contact interactions such as embrace, carry, or grapple. The benchmark provides a 2,500-sample test set with 14 interaction categories and four contact densities, and scores outputs on six axes including anatomy and interaction via mesh reconstruction.","whyItMatters":"Existing evaluations often overlook anatomical and geometric errors in multi-person editing, and VLM-based judges may rate such errors as acceptable. This benchmark provides a geometry-based scoring method that tracks human judgment more closely, enabling more reliable comparisons of editing models on this challenging task.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7611bdd34d173844a0f058dc43ffaa34f24a05879d32efb35e197280ec69d0df"},"motivation":"Text-to-image and personalized editing models now synthesize high-fidelity single-subject images with ease.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27616","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MPIE-Bench Team","organizationType":"benchmark-organization","sourceUrl":"https://github.com/AnnLin0628/mpie-bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mmooc_7beeb312","familyId":"bmf_4c8571bb6649","name":"MMOOC","oneLine":"MMOOC evaluates multimodal large language models on out-of-context (OOC) and shifted in-context (Shifted IC) visual question answering. It contains over 41K image-question pairs covering three question formats, eight shift types, and six visual scenarios. Responses are scored for accuracy and refusal rate, with an LLM-as-a-judge metric for reasoning correctness.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27637","pdf":"https://arxiv.org/pdf/2607.27637","project":"https://zhuwenjie98.github.io/MMOOC-project-page/","code":"https://github.com/ZhuWenjie98/MMOOC","data":null,"hfPaper":"https://huggingface.co/papers/2607.27637"},"evidence":{"snippet":"To fill this gap, we present MMOOC, a large-scale benchmark for evaluating refusal and robust answering abilities of MLLMs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":6,"hfDailySubmittedAt":"2026-08-11T00:00:00.000Z","githubStars":31,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27637"},"ranking":{"30d":{"score":75,"rank":18,"coverage":0.85,"confidence":"High"},"90d":{"score":69,"rank":78,"coverage":0.7,"confidence":"Medium"}},"description":"MMOOC evaluates multimodal large language models on out-of-context (OOC) and shifted in-context (Shifted IC) visual question answering. It contains over 41K image-question pairs covering three question formats, eight shift types, and six visual scenarios. Responses are scored for accuracy and refusal rate, with an LLM-as-a-judge metric for reasoning correctness.","whyItMatters":"Existing benchmarks focus on unanswerable questions but overlook answerable shifted contexts. MMOOC provides a joint evaluation of refusal and robust answering, offering insight into model reliability in real-world scenarios where contexts are imperfect. This supports comparing models on a balanced measure of capability and safety.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"185953f78351ad4e6fd1aec2a3adc8e362b4ffa5bcc00ab38aa66c6f6989464e"},"motivation":"Multimodal Large Language Models (MLLMs) have achieved strong performance on a wide range of vision-language tasks, but often fail under imperfect or shifted contexts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27637","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MMOOC Project Team","organizationType":"academic-lab","sourceUrl":"https://zhuwenjie98.github.io/MMOOC-project-page/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mmhbench_02d53b0e","familyId":"bmf_756e91e3ea85","name":"MMHBench","oneLine":"MMHBench is a multimodal benchmark for mental health understanding in long-form videos, comprising 268 videos and 2,184 questions across third-person and first-person settings. It evaluates model reasoning about mental states from multimodal evidence.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27895","pdf":"https://arxiv.org/pdf/2607.27895","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.27895"},"evidence":{"snippet":"To address this limitation, we introduce MMHBench, a comprehensive multimodal benchmark for multi-perspective mental health understanding, comprising 268 long-form videos and 2,184 carefully curated questions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27895"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MMHBench is a multimodal benchmark for mental health understanding in long-form videos, comprising 268 videos and 2,184 questions across third-person and first-person settings. It evaluates model reasoning about mental states from multimodal evidence.","whyItMatters":"This benchmark targets a critical domain where existing evaluation is limited to coarse classification. It could provide finer-grained assessment of mental health understanding, but its current availability is unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9c8f40c7af40d058f987f2d913e6c9886c254aa59af9ff397c26f194785ee21b"},"motivation":"Mental health understanding in long-form videos requires nuanced reasoning over observable behavior, interpersonal context, and latent psychological states.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27895","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_migue-bench_dbd75fa4","familyId":"bmf_9437ac27c8a9","name":"MiGUE-Bench","oneLine":"MiGUE-Bench is a benchmark for multi-granularity event analysis, covering event detection, relation reasoning, structure induction, and future prediction across single to cross-document settings. It uses an LLM-driven annotation pipeline.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27654","pdf":"https://arxiv.org/pdf/2607.27654","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.27654"},"evidence":{"snippet":"To address these limitations, we introduce MiGUE-Bench, a systematic benchmark for assessing the performance of LLMs in multi-granularity event analysis.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27654"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MiGUE-Bench is a benchmark for multi-granularity event analysis, covering event detection, relation reasoning, structure induction, and future prediction across single to cross-document settings. It uses an LLM-driven annotation pipeline.","whyItMatters":"Event analysis spans diverse tasks at different document granularity, and this benchmark aims to systematically assess LLM capabilities across them. Its breadth could inform progress in information extraction.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9ac34bd126caeaece8c760ab62c2a66247d8eb556cb6efaf9322ef7e2da1c7aa"},"motivation":"Event analysis is an essential and fundamental direction of information extraction, involving various event-centric tasks at different granularity of documents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"publication_reported","venue":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '26), pp. 3464-3472, 2026","evidence":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '26), pp. 3464-3472, 2026","evidenceUrl":"https://arxiv.org/abs/2607.27654","source":"arxiv-journal-reference","evidenceLevel":"strong-author-metadata","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publications":[{"venueName":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '26), pp. 3464-3472, 2026","publicationStatus":"published","evidence":[{"sourceType":"arxiv-journal-reference","sourceUrl":"https://arxiv.org/abs/2607.27654","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '26), pp. 3464-3472, 2026","level":"strong-author-metadata"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_memebench_baa40d97","familyId":"bmf_6bf115ae11a0","name":"MemeBench","oneLine":"MemeBench is a diagnostic benchmark of 1,253 Chinese and English memes with human-written references and VIKR annotations—Visual clues, Identity links, Knowledge units, and Reasoning mechanisms—for evaluating interpretation in LVLMs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27798","pdf":"https://arxiv.org/pdf/2607.27798","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.27798"},"evidence":{"snippet":"We introduce MemeBench, a diagnostic benchmark of 1,253 Chinese and English memes with human-written references and quality-controlled VIKR annotations, centered on anime, comics, games, and adjacent online subcultures.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27798"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MemeBench is a diagnostic benchmark of 1,253 Chinese and English memes with human-written references and VIKR annotations—Visual clues, Identity links, Knowledge units, and Reasoning mechanisms—for evaluating interpretation in LVLMs.","whyItMatters":"Memes rely on cultural knowledge beyond visual content, and this benchmark attempts to decompose interpretation into components. It could help identify gaps in LVLM understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a249a484afb479f5ad284f292969923fcbab754df104ef3581f8bdb734f0fce0"},"motivation":"Large vision-language models have improved at describing visual content, but accurate descriptions do not ensure interpretation when meaning depends on knowledge beyond the pixels.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27798","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_lomevqa_dd681f2a","familyId":"bmf_1b89ee4a1e86","name":"LoMeVQA","oneLine":"LoMeVQA is a benchmark for longitudinal medical visual question answering, with 206K VQA pairs across five tasks: progress classification, progress description, progress report generation, differential region grounding, and differential region description.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27806","pdf":"https://arxiv.org/pdf/2607.27806","project":null,"code":"https://github.com/pepperbubble/LoMeVQA","data":null,"hfPaper":"https://huggingface.co/papers/2607.27806"},"evidence":{"snippet":"To fill this gap, we propose LoMeVQA, a comprehensive benchmark consisting of 206K longitudinal visual question answering (VQA) pairs for temporal medical image analysis.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27806"},"ranking":{"30d":{"score":46,"rank":85,"coverage":0.85,"confidence":"High"},"90d":{"score":39,"rank":267,"coverage":0.7,"confidence":"Medium"}},"description":"LoMeVQA is a benchmark for longitudinal medical visual question answering, with 206K VQA pairs across five tasks: progress classification, progress description, progress report generation, differential region grounding, and differential region description.","whyItMatters":"Longitudinal medical reasoning is underexplored in MLLMs, and this benchmark provides a comprehensive testbed. It reveals limitations in temporal reasoning and supports future improvements in medical AI.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"aa1e98156bb41f79638f8dde30a86dd02a23d36af052749bfc0c7b822d960fa5"},"motivation":"In clinical practice, patients often undergo multiple imaging examinations over successive visits, yielding longitudinal data.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27806","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LoMeVQA Project Team","organizationType":"academic-lab","sourceUrl":"https://github.com/pepperbubble/LoMeVQA","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_layoutbench_a9f59f74","familyId":"bmf_f115fe1b3e4f","name":"LayoutBench","oneLine":"LayoutBench evaluates three cloud storage layout strategies (individual objects, tar archives, Parquet columns) for multimedia data retrieval, measuring retrieval time, data transferred, and cost.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.DC"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.28880","pdf":"https://arxiv.org/pdf/2607.28880","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.28880"},"evidence":{"snippet":"We present LayoutBench, the first benchmark designed to fill this gap.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.28880"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LayoutBench evaluates three cloud storage layout strategies (individual objects, tar archives, Parquet columns) for multimedia data retrieval, measuring retrieval time, data transferred, and cost.","whyItMatters":"This benchmark fills a gap in storage benchmarking for multimedia workloads, but it is not a model evaluation benchmark.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"99c00b3ab4c9c4fa24a37e19392ec8e41f9cef5d8f60b322fd98783f671193e6"},"motivation":"Modern multimedia machine learning workloads increasingly store large-scale datasets in cloud object storage services such as AWS S3.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.28880","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_infoops-bench_c0a89e01","familyId":"bmf_700e7acd1992","name":"InfoOps Bench","oneLine":"InfoOps Bench is an active, constantly updated benchmark measuring the integrity of frontier language models against co-optation for information operations. It uses real examples from a live monitoring pipeline and tests 17 models from 8 providers.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.28503","pdf":"https://arxiv.org/pdf/2607.28503","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.28503"},"evidence":{"snippet":"In this paper we present an active, constantly updated AI benchmark which measures the integrity of frontier language models against being co-opted for use by authoritarian state \"information operations\": intentional, coordinated activities by one state to influence public opinion and information ecosystems in another state.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.28503"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"InfoOps Bench is an active, constantly updated benchmark measuring the integrity of frontier language models against co-optation for information operations. It uses real examples from a live monitoring pipeline and tests 17 models from 8 providers.","whyItMatters":"This benchmark addresses a novel safety concern: whether models can be co-opted for state-sponsored information operations. It could drive improvements in model integrity and safety.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"da7894add9a7d0713e21a23f26af308f20ff76880cec5a18ae35ce6dae497909"},"motivation":"In this paper we present an active, constantly updated AI benchmark which measures the integrity of frontier language models against being co-opted for use by authoritarian state \"information operations\": intentional, coordinated activities by one state to influence public opinion and information ecosystems in another state.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.28503","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_ifhierbench_c5c33d5e","familyId":"bmf_ed9dc64cf457","name":"IFHierBench","oneLine":"IFHierBench is a hierarchical instruction-following benchmark with 600 prompts and deterministic checkers, evaluating LLMs on satisfying constraints at different output scopes. It measures prompt-level accuracy.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27912","pdf":"https://arxiv.org/pdf/2607.27912","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.27912"},"evidence":{"snippet":"We introduce IFHierBench, a hierarchical instruction-following benchmark of 600 prompts stratified across four constraint-tree depths and 35 distinct constraints, each prompt paired with a deterministic checker that verifies satisfaction at every scope.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27912"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"IFHierBench is a hierarchical instruction-following benchmark with 600 prompts and deterministic checkers, evaluating LLMs on satisfying constraints at different output scopes. It measures prompt-level accuracy.","whyItMatters":"Instruction-following is critical for LLM deployment, and existing benchmarks treat constraints flatly. This benchmark addresses a gap by evaluating nested constraints, but its availability is unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"970ea146b7affeba1c5fb611bbbad1463fc3ad2523d37a82ce16266a53fa58c7"},"motivation":"Instruction-following ability is critical for deploying large language models in real-world applications, where downstream components depend on the output satisfying specific constraints.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27912","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_eeg-editbench_8f55ec4b","familyId":"bmf_23b204771077","name":"EEG-EditBench","oneLine":"EEG-EditBench evaluates EEG-to-image retrieval models using 2,137 controlled edits of 200 THINGS-EEG2 test images, covering object identity, attributes, background, and presence, with metrics like 200-way accuracy and 2AFC accuracy.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CV"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.27857","pdf":"https://arxiv.org/pdf/2607.27857","project":null,"code":"https://github.com/XiaoZhangYES/EEG-EditBench","data":"https://huggingface.co/datasets/xiaozgg/EEG-EditBench","hfPaper":"https://huggingface.co/papers/2607.27857"},"evidence":{"snippet":"Motivated by this question, we introduce EEG-EditBench, a diagnostic benchmark that examines this question through controlled edits of object identity, attributes, background, and object presence.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":819,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2607.27857"},"ranking":{"30d":{"score":55,"rank":56,"coverage":0.7,"confidence":"Medium","datasetDownloadRank":4,"datasetRankPopulation":20},"90d":{"score":48,"rank":211,"coverage":0.85,"confidence":"High","datasetDownloadRank":10,"datasetRankPopulation":51}},"description":"EEG-EditBench evaluates EEG-to-image retrieval models using 2,137 controlled edits of 200 THINGS-EEG2 test images, covering object identity, attributes, background, and presence, with metrics like 200-way accuracy and 2AFC accuracy.","whyItMatters":"Standard retrieval accuracy can mask whether models truly preserve visual information. EEG-EditBench provides a controlled, repeatable protocol for probing fine-grained visual distinctions, informing development of more robust EEG decoding models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"424af334db84c36047b41a0ae784bbd71965283c112866750ab2cca8e4125c80"},"motivation":"Recent EEG-to-image retrieval models have achieved strong performance in identifying viewed images from semantically diverse candidates.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27857","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Search & Retrieval"],"domainScope":"general"},{"id":"bm_earlydx_668e5f94","familyId":"bmf_8ad7d2d6cad1","name":"EarlyDx","oneLine":"EarlyDx is a benchmark for open-ended early diagnosis from emergency department encounters in MIMIC-IV, using admission-time records and LLM-auditor-supervised free-text labels.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.28788","pdf":"https://arxiv.org/pdf/2607.28788","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.28788"},"evidence":{"snippet":"We introduce EarlyDx, a large-scale benchmark for open-ended early diagnosis, built from 154,834 emergency department encounters in MIMIC-IV.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.28788"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EarlyDx is a benchmark for open-ended early diagnosis from emergency department encounters in MIMIC-IV, using admission-time records and LLM-auditor-supervised free-text labels.","whyItMatters":"It addresses evaluation of diagnosis prediction under realistic admission constraints, which existing closed-set benchmarks miss.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cdf7b32c5e6afa906bb2eaca16c826fdf1ad6a12315b6799b6f505e0b0dcd34c"},"motivation":"Clinical diagnosis at hospital admission must be made rapidly from limited, incomplete evidence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.28788","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_dataclaweval_e7e41e17","familyId":"bmf_2921fe8e003f","name":"DataClawEval","oneLine":"DataClawEval evaluates autonomous data-engineering agents across 100 end-to-end tasks spanning PySpark, MySQL, HiveSQL, PrestoSQL/Trino, and FlinkSQL, with deterministic rule-based grading in isolated sandboxes.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.28033","pdf":"https://arxiv.org/pdf/2607.28033","project":null,"code":"https://github.com/Dicemy/DataClawEval/tree/master","data":null,"hfPaper":"https://huggingface.co/papers/2607.28033"},"evidence":{"snippet":"To bridge this gap, we introduce DataClawEval, the first comprehensive benchmark designed specifically to evaluate the end-to-end task completion capabilities of autonomous agents in real-world data engineering scenarios.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"hosting_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.28033"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"DataClawEval evaluates autonomous data-engineering agents across 100 end-to-end tasks spanning PySpark, MySQL, HiveSQL, PrestoSQL/Trino, and FlinkSQL, with deterministic rule-based grading in isolated sandboxes.","whyItMatters":"It provides a reproducible, deterministic evaluation for industrial data-engineering workflows, revealing domain-specific strengths and gaps in agent capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"55f381818930f984f7418761c80a05f58e2c49e9f42dbde2060ab18d3f64d1e9"},"motivation":"Large language models (LLMs) and LLM-based agents are increasingly being deployed to automate complex workflows, promising to revolutionize data management and processing.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.28033","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DataClawEval Team","organizationType":"community","sourceUrl":"https://github.com/Dicemy/DataClawEval/tree/master","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_cxr-retrieve_95ed172c","familyId":"bmf_7d27f3758eea","name":"CXR-Retrieve","oneLine":"CXR-Retrieve is a benchmark for compositional chest X-ray text-to-image retrieval, with 5,159 MIMIC-CXR-JPG test images and 145 queries over single and conjunction findings with positive and negative assertions.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Information retrieval"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27779","pdf":"https://arxiv.org/pdf/2607.27779","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.27779"},"evidence":{"snippet":"This creates an objective mismatch: a model may retrieve images related to words in the query while failing to satisfy the full clinical constraint, especially for conjunctions and negations such as ``atelectasis and no pneumonia.'' We introduce CXR-Retrieve, a structured benchmark for compositional chest X-ray text-to-image retrieval.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27779"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CXR-Retrieve is a benchmark for compositional chest X-ray text-to-image retrieval, with 5,159 MIMIC-CXR-JPG test images and 145 queries over single and conjunction findings with positive and negative assertions.","whyItMatters":"Addresses the gap between report-to-image matching and complex clinical query satisfaction, crucial for real-world search reliability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"12143ad48a34ee4266801b01b4cb63e6bac96d9a1e0f657123ab864b3f8f11ed"},"motivation":"Large chest radiography archives are difficult to search because most studies are paired only with free-text reports rather than structured clinical annotations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27779","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Search & Retrieval"],"domainScope":"specific"},{"id":"bm_clawtrack_7bde407b","familyId":"bmf_486094563286","name":"ClawTrack","oneLine":"ClawTrack is a dual-assessment benchmark for agents, measuring task outcomes and process quality across 320 tasks in 8 domains with 25+ mock services.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.28037","pdf":"https://arxiv.org/pdf/2607.28037","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.28037"},"evidence":{"snippet":"In this work, we present ClawTrack, a dual-assessment benchmark that simultaneously measures what an agent achieves (Task Score) and how it achieves it (Process Score).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.28037"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"ClawTrack is a dual-assessment benchmark for agents, measuring task outcomes and process quality across 320 tasks in 8 domains with 25+ mock services.","whyItMatters":"It aims to decompose agent success into reasoning dimensions, which could improve attribution and post-training filtering.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2a8741ac7160ee81fa9e4468214f463d5efa3b45ef56d79b3b01a14e5b460470"},"motivation":"As LLM-based agents are deployed in complex, multi-step workflows, a critical evaluation gap has emerged: most existing benchmarks judge only final outcomes, unable to distinguish reliable reasoning from lucky success or attribute failures to specific process deficiencies, hindering attribution in long-horizon tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.28037","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_chem-world_1eaa9499","familyId":"bmf_e29ccb36b6a1","name":"Chem World","oneLine":"Chem World integrates 17 chemical datasets with 800,000+ molecules for property prediction, offering a unified evaluation platform.","area":"Language & Knowledge","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.28079","pdf":"https://arxiv.org/pdf/2607.28079","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.28079"},"evidence":{"snippet":"In this work, we introduce Chem World, a comprehensive benchmark for chemical property prediction that integrates 17 diverse chemical datasets with over 800,000 molecular samples, covering various properties including density, electrical conductivity, solubility, and other molecular characteristics.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.28079"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Chem World integrates 17 chemical datasets with 800,000+ molecules for property prediction, offering a unified evaluation platform.","whyItMatters":"It aims to standardize chemical property prediction evaluation, improving reliability and comparison across models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8aef31d0c752c70a4321f2fc42f7baf570478940e7f49798346670f57bfd3bae"},"motivation":"Chemical property prediction plays a critical role in accelerating scientific discovery in chemistry, materials science, and drug development.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.28079","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_vamos-bench_83d46046","familyId":"bmf_e925739160ae","name":"VAmoS Bench","oneLine":"VAmoS Bench evaluates complete voice-agent systems in a stateful customer-support task, with 100 scenarios, a simulated caller, real SQL tools, and binary assertions assessed against full interaction traces.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27453","pdf":"https://arxiv.org/pdf/2607.27453","project":null,"code":"https://github.com/veris-ai/riley-agent","data":null,"hfPaper":"https://huggingface.co/papers/2607.27453"},"evidence":{"snippet":"To address this gap, we introduce VAmoS Bench, the Voice Agent Simulation Bench.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":16,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27453"},"ranking":{"30d":{"score":69,"rank":31,"coverage":0.55,"confidence":"Low"},"90d":{"score":62,"rank":123,"coverage":0.55,"confidence":"Low"}},"description":"VAmoS Bench evaluates complete voice-agent systems in a stateful customer-support task, with 100 scenarios, a simulated caller, real SQL tools, and binary assertions assessed against full interaction traces.","whyItMatters":"It measures end-to-end call containment and correct backend mutations, going beyond component metrics to capture task-level correctness in realistic scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7f0d7e4077e7b8b6c367032c3d08c46f652f9b532fc516ef18eeac0ab9c02205"},"motivation":"Production voice agents span cascaded, speech-to-speech, and hybrid architectures.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27453","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Veris AI","organizationType":"company-research-lab","sourceUrl":"https://github.com/veris-ai/riley-agent","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_treat_ece84fbf","familyId":"bmf_a5075550e1f2","name":"TREAT","oneLine":"TREAT evaluates LLMs' ability to recognize theorem identities from equivalence-preserving formula transformations, with 737 identities and 29,480 transformed rows.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07540","pdf":"https://arxiv.org/pdf/2608.07540","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07540"},"evidence":{"snippet":"We introduce TREAT, a benchmark for evaluating whether large language models can recover known theorem identities from equivalence-preserving formula-level transformations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07540"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TREAT evaluates LLMs' ability to recognize theorem identities from equivalence-preserving formula transformations, with 737 identities and 29,480 transformed rows.","whyItMatters":"It targets representation-robust access to formal knowledge, critical for AI tools interacting with formal systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"858fffa6bbbcf8b0a17836ac548bed4a1083e14912b97c353b2448955ad8e95e"},"motivation":"AI systems increasingly operate between flexible input representations and formal objects used by downstream tools.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"28th International Symposium on Symbolic and Numeric Algorithms for Scientific Computing (SYNASC 2026)","evidence":"Accepted at 28th International Symposium on Symbolic and Numeric Algorithms for Scientific Computing (SYNASC 2026)","evidenceUrl":"https://arxiv.org/abs/2608.07540","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"28th International Symposium on Symbolic and Numeric Algorithms for Scientific Computing (SYNASC 2026)","reviewStatus":"accepted","decisionRaw":"Accepted at 28th International Symposium on Symbolic and Numeric Algorithms for Scientific Computing (SYNASC 2026)","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.07540","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at 28th International Symposium on Symbolic and Numeric Algorithms for Scientific Computing (SYNASC 2026)","level":"author-claim"}]}],"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_toxscreen_d4a7771e","familyId":"bmf_fa2e802a8fcc","name":"ToxScreen","oneLine":"A benchmark of roughly 800 backdoored LLMs across attack objectives, trigger mechanisms, poisoning rates, model scales, and training mechanisms. Evaluates whether a defender can recover a planted trigger given white-box access and behavior of concern.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CR"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.26849","pdf":"https://arxiv.org/pdf/2607.26849","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.26849"},"evidence":{"snippet":"To evaluate whether a defender can recover such a trigger under realistic settings, we release ToxScreen, a benchmark of roughly 800 backdoored models spanning attack objectives, trigger mechanisms, poisoning rates, model scales, and backdoor training mechanisms.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26849"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark of roughly 800 backdoored LLMs across attack objectives, trigger mechanisms, poisoning rates, model scales, and training mechanisms. Evaluates whether a defender can recover a planted trigger given white-box access and behavior of concern.","whyItMatters":"Backdoor recovery in LLMs is a critical security challenge; this benchmark provides a standardized testbed for comparing trigger-recovery methods under realistic constraints, informing practical defense strategies.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"449dfacec19ae637c5913cadd482413d54fabbfb92fb1e6ba5812945d34da3be"},"motivation":"As large language models (LLMs) are deployed in high-stakes domains, adversaries may poison training data to implant backdoors: hidden triggers that covertly manipulate model behavior at inference time.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26849","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_swe-nfi_7cf5877f","familyId":"bmf_b14a6af93273","name":"SWE-NFI","oneLine":"A benchmark of 188 tasks for evaluating coding agents on non-functional improvements in Python projects, with 92 executable rules combining functional correctness and rule-based evaluation.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27409","pdf":"https://arxiv.org/pdf/2607.27409","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.27409"},"evidence":{"snippet":"In this paper, we present SWE-NFI, a benchmark for evaluating coding agents on NFIs beyond functional correctness.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27409"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark of 188 tasks for evaluating coding agents on non-functional improvements in Python projects, with 92 executable rules combining functional correctness and rule-based evaluation.","whyItMatters":"Existing coding benchmarks focus on functional correctness; this benchmark addresses the gap in evaluating behavior-preserving code quality improvements, useful for assessing real-world software engineering capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5f29664e18b98d6ce93447ff56a9a4f0b4ba0d9b77625f5d60cddf5dbc5d0c25"},"motivation":"Although coding agents have achieved impressive performance on correctness-oriented benchmarks, their ability to make behavior-preserving non-functional improvements (NFIs) remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27409","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_statmechbench-v0_d1047ef0","familyId":"bmf_d86596b80f05","name":"StatMechBench-v0","oneLine":"A benchmark of six Ising-type problems for evaluating LLM agents' ability to discover statistical mechanical mappings from raw partition functions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.26367","pdf":"https://arxiv.org/pdf/2607.26367","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.26367"},"evidence":{"snippet":"To probe this question, we introduce StatMechBench-v0, a benchmark of six Ising-type problems covering transfer-matrix methods, gauge-removable disorder, and planar/Pfaffian structure.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26367"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark of six Ising-type problems for evaluating LLM agents' ability to discover statistical mechanical mappings from raw partition functions.","whyItMatters":"This probes AI capabilities for structural discovery in theoretical physics, highlighting limitations in current LLM reasoning and the need for verification beyond numerical agreement.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ef72598a7d2b5379357e9526d30e81fee1c3b988fbaa7039df7bf54ef8526974"},"motivation":"An important skill in theoretical physics is to recognize when a new problem can be transformed into a known model.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"AID-Wild workshop at CAIS 2026","evidence":"Accepted to the AID-Wild workshop at CAIS 2026","evidenceUrl":"https://arxiv.org/abs/2607.26367","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"AID-Wild workshop at CAIS 2026","reviewStatus":"accepted","decisionRaw":"Accepted to the AID-Wild workshop at CAIS 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.26367","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to the AID-Wild workshop at CAIS 2026","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_setoka_2726d0e3","familyId":"bmf_eec5c354913b","name":"Setoka","oneLine":"A benchmark for evaluating memory-augmented personalized agents on hierarchical user understanding from heterogeneous data, with four levels of user understanding and psychometrics-based synthetic data.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27056","pdf":"https://arxiv.org/pdf/2607.27056","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.27056"},"evidence":{"snippet":"In this work, we propose Setoka, a benchmark for evaluating memory-augmented personalized agents with hierarchical user understanding from heterogeneous data.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27056"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark for evaluating memory-augmented personalized agents on hierarchical user understanding from heterogeneous data, with four levels of user understanding and psychometrics-based synthetic data.","whyItMatters":"Personalized agents require deeper user understanding beyond fact retrieval; this benchmark provides a standard evaluation for cross-source integration and abstraction over long-term user behavior.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"709be3da37b7b0e950912e3ba256c14f0f48267d794cf70629dec6aa18a2ac20"},"motivation":"Personalized agents are increasingly applied to assist users across a wide range of tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27056","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_secrespond_a48429f2","familyId":"bmf_9705c5bae016","name":"SecRespond","oneLine":"A benchmark for evaluating LLM agents on post-compromise incident-response workflows using forensic disk snapshots and host security reports across 10 cyber ranges.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.26791","pdf":"https://arxiv.org/pdf/2607.26791","project":null,"code":"https://github.com/Alibaba-NLP/qqr/tree/main/data/secrespond","data":null,"hfPaper":"https://huggingface.co/papers/2607.26791"},"evidence":{"snippet":"To address this gap, we introduce SecRespond, the first benchmark for evaluating LLM agents on the post-compromise incident-response workflow.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":4,"hfDailySubmittedAt":"2026-07-30T00:00:00.000Z","githubStars":280,"githubScope":"hosting_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26791"},"ranking":{"30d":{"score":51,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":52,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"A benchmark for evaluating LLM agents on post-compromise incident-response workflows using forensic disk snapshots and host security reports across 10 cyber ranges.","whyItMatters":"Cybersecurity benchmarks typically focus on pre-compromise settings; this addresses the gap in evaluating agents for real-world incident response, where proactive investigation and remediation are critical.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6f9ef85dd26044adf9cfbb4910f7395137fb5d60b1f2112ec1964a99e2556f2f"},"motivation":"Large Language Model (LLM) agents are increasingly adopted in real-world security operations with access to host artifacts and command-line interfaces (CLIs), making it critical to thoroughly assess their security capabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26791","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Alibaba NLP","organizationType":"company-research-lab","sourceUrl":"https://github.com/Alibaba-NLP/qqr/tree/main/data/secrespond","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_scifigqual-bench_f473d66d","familyId":"bmf_dc38c7b96fa2","name":"SciFigQual-Bench","oneLine":"A benchmark for evaluating scientific figure quality across five dimensions with full-manuscript context, using 6,308 expert-rated images from top CS conferences and a fixed eval1200 test split.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27084","pdf":"https://arxiv.org/pdf/2607.27084","project":"https://frankdengai.github.io/SciFigQual-Bench","code":"https://github.com/FrankDengAI/SciFigQual-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.27084"},"evidence":{"snippet":"To address this, we propose SciFigQual-Bench, a full-text contextual benchmark that evaluates scientific images across five dimensions (clarity, layout, caption fit, context relevance, and misleading risk).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27084"},"ranking":{"30d":{"score":24,"rank":143,"coverage":0.85,"confidence":"High"},"90d":{"score":25,"rank":387,"coverage":0.7,"confidence":"Medium"}},"description":"A benchmark for evaluating scientific figure quality across five dimensions with full-manuscript context, using 6,308 expert-rated images from top CS conferences and a fixed eval1200 test split.","whyItMatters":"Existing image quality assessment methods are unsuitable for scientific figures; this benchmark provides a contextual, multi-dimensional standard for automated evaluation, enabling model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e36f909aaba8edc3ef929e6ba96e415ccd670756f5472f6d6e9e90fc4ed504a9"},"motivation":"Scientific images are the core elements of presenting experimental conclusions, elaborating system architecture, and supporting comparative arguments in scientific papers.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27084","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FrankDengAI","organizationType":"academic-lab","sourceUrl":"https://github.com/FrankDengAI/SciFigQual-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_safebuild-bench_538eb6ea","familyId":"bmf_e22800c63b0a","name":"SafeBuild-Bench","oneLine":"A benchmark for evaluating multimodal large language models on construction safety hazard identification and description, with 3,314 expert-verified task instances from over 3,000 images.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal","Safety"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00068","pdf":"https://arxiv.org/pdf/2608.00068","project":null,"code":"https://github.com/safebuild/gems","data":null,"hfPaper":"https://huggingface.co/papers/2608.00068"},"evidence":{"snippet":"We introduce SafeBuild-Bench, a metadata-driven benchmark for evaluating multimodal large language models on construction safety under realistic temporal and site variation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00068"},"ranking":{"30d":{"score":46,"rank":86,"coverage":0.85,"confidence":"High"},"90d":{"score":39,"rank":268,"coverage":0.7,"confidence":"Medium"}},"description":"A benchmark for evaluating multimodal large language models on construction safety hazard identification and description, with 3,314 expert-verified task instances from over 3,000 images.","whyItMatters":"Construction-safety models must handle realistic temporal and site variation; this benchmark provides a standard evaluation for hazard recognition and description, useful for deployment risk assessment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8131bb22ab159a7d073ef4a03c6739f0a5bc5ebf8348ea3d6b241631f5309bb0"},"motivation":"Construction-safety models must handle concrete deployment risks, such as a worker standing near a scaffold edge without guardrails, rather than only recognize common objects in curated images.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"KDD 2026","evidence":"Accepted by KDD 2026. 12 pages, 6 figures","evidenceUrl":"https://arxiv.org/abs/2608.00068","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"KDD 2026","reviewStatus":"accepted","decisionRaw":"Accepted by KDD 2026. 12 pages, 6 figures","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2608.00068","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by KDD 2026. 12 pages, 6 figures","level":"author-claim"}]}],"publishers":[{"name":"SafeBuild","organizationType":"academic-lab","sourceUrl":"https://github.com/safebuild/gems","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_pause_e497d85d","familyId":"bmf_6210c0bf0539","name":"PAUSE","oneLine":"A user-centric benchmark for evaluating personal AI assistants in stateful, service-integrated environments, with multi-regime evaluation and user simulation for long-horizon tasks.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27354","pdf":"https://arxiv.org/pdf/2607.27354","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.27354"},"evidence":{"snippet":"We introduce PAUSE, a user-centric benchmark for evaluating personal AI assistants in stateful, service-integrated environments.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27354"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A user-centric benchmark for evaluating personal AI assistants in stateful, service-integrated environments, with multi-regime evaluation and user simulation for long-horizon tasks.","whyItMatters":"Personal AI assistants need to handle stateful, user-configuration-aware interactions across services; this benchmark provides a framework for evaluating user-centric performance in realistic settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2e4413afc06534891930cb8cda1d3f06664b6536d382ee0fee37e3357bff47df"},"motivation":"Personal AI assistants are increasingly deployed as task-oriented, tool-augmented agents that operate within unified service environments to support everyday user activities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27354","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_ovearth-bench_169abada","familyId":"bmf_d6371bc0182b","name":"OVEarth-Bench","oneLine":"Evaluates open-vocabulary Earth observation models on category breadth and query diversity, covering mask and box localization across vocabulary, referring, and reasoning queries under a unified zero-shot protocol.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.27278","pdf":"https://arxiv.org/pdf/2607.27278","project":"https://earth-insights.github.io/OVEarth-bench","code":"https://github.com/earth-insights/OVEarth-bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.27278"},"evidence":{"snippet":"To fill this gap, we introduce OVEarth-Bench, which extends existing evaluation in two directions: category breadth, through broad hierarchical category coverage with positive and negative expressions, and query diversity, through vocabulary, referring, and reasoning queries.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":15,"hfDailySubmittedAt":"2026-07-30T00:00:00.000Z","githubStars":11,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27278"},"ranking":{"30d":{"score":75,"rank":20,"coverage":0.85,"confidence":"High"},"90d":{"score":64,"rank":109,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates open-vocabulary Earth observation models on category breadth and query diversity, covering mask and box localization across vocabulary, referring, and reasoning queries under a unified zero-shot protocol.","whyItMatters":"Existing EO benchmarks cover limited categories and query forms, making it hard to gauge real-world capability. This benchmark provides a broader, more diverse evaluation to compare general and EO-specific models on open-vocabulary localization.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"853b814d4a592b1b15804629d2764233a3159622e1246be4e448e1bd3f2d1769"},"motivation":"Open-vocabulary Earth observation (EO) aims to localize geospatial concepts specified in natural language rather than a fixed label set.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27278","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Earth Insights","organizationType":"academic-lab","sourceUrl":"https://earth-insights.github.io/OVEarth-bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_omegause-officeval_6b76ef98","familyId":"bmf_7046d745096e","name":"OmegaUse-OfficeVal","oneLine":"Evaluates LLM agents on long-horizon office-suite tasks from 100 practitioner-derived scenarios, scoring deliverable quality through code-based verifiers and comparing performance against economic signals of human labor time and price.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27155","pdf":"https://arxiv.org/pdf/2607.27155","project":"https://omegause-officeval.github.io","code":"https://github.com/baidu-frontier-research/OmegaUse-OfficeVal","data":null,"hfPaper":"https://huggingface.co/papers/2607.27155"},"evidence":{"snippet":"We introduce OmegaUse-OfficeVal, a benchmark for evaluating LLM agents on long-horizon office-suite tasks with task-level economic grounding.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":13,"hfDailySubmittedAt":"2026-07-30T00:00:00.000Z","githubStars":8,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27155"},"ranking":{"30d":{"score":73,"rank":25,"coverage":0.85,"confidence":"High"},"90d":{"score":60,"rank":134,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates LLM agents on long-horizon office-suite tasks from 100 practitioner-derived scenarios, scoring deliverable quality through code-based verifiers and comparing performance against economic signals of human labor time and price.","whyItMatters":"Existing agent benchmarks rarely consider cost-effectiveness for real office workflows. This benchmark provides a way to compare agent output quality relative to human labor costs, supporting value-weighted evaluation of office automation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2bad570f995c8afbc6c349d2793cf84552615b7dc98214ae191a25d8c28c6e43"},"motivation":"Large language model (LLM) agents are increasingly expected to assist users in completing tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27155","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Baidu Frontier Research","organizationType":"company-research-lab","sourceUrl":"https://github.com/baidu-frontier-research/OmegaUse-OfficeVal","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_multivationbench_9e6dc7dd","familyId":"bmf_7a98442d82f1","name":"MultivationBench","oneLine":"Evaluates multimodal motivation reasoning in story-driven visual narratives, using 1,000 narratives with 16,092 multi-label questions grounded in Maslow and Reiss frameworks across definition and practical reasoning tasks.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.26465","pdf":"https://arxiv.org/pdf/2607.26465","project":null,"code":"https://github.com/HKUST-KnowComp/MultivationBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.26465"},"evidence":{"snippet":"To address this gap, we introduce MultivationBench, a benchmark designed to rigorously evaluate multimodal motivation reasoning within story-driven visual narratives.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26465"},"ranking":{"30d":{"score":50,"rank":75,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":284,"coverage":0.55,"confidence":"Low"}},"description":"Evaluates multimodal motivation reasoning in story-driven visual narratives, using 1,000 narratives with 16,092 multi-label questions grounded in Maslow and Reiss frameworks across definition and practical reasoning tasks.","whyItMatters":"Assesses whether models can reason about evolving character motivations across sequential context, an underexplored capability for social intelligence.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"19bb77e91f48cd998c6b16c6cafab07a4b4d419e86fbe7ef854ff7182322a875"},"motivation":"Multimodal Large Language Models have sparked significant interest due to their potential for social intelligence; however, their ability to perform sequential motivation reasoning remains insufficiently studied.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26465","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_memsecbench_ec92c168","familyId":"bmf_d73b74353527","name":"MemSecBench","oneLine":"Evaluates lifecycle security of agent memory systems with 310 cases across 48 contexts, using a Write-Execute-Forget protocol and evidence-based adjudication across seven checkpoints.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27080","pdf":"https://arxiv.org/pdf/2607.27080","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.27080"},"evidence":{"snippet":"To address this gap, we introduce MemSecBench, a task-grounded benchmark for the lifecycle security of agent memory systems.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27080"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates lifecycle security of agent memory systems with 310 cases across 48 contexts, using a Write-Execute-Forget protocol and evidence-based adjudication across seven checkpoints.","whyItMatters":"Provides insight into how malicious instructions can persist in memory systems and affect later actions, highlighting security differences across configurations.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"019db75a2085515681bf1d13548a7b136775d006cf368ae4f9dd9963d73e8796"},"motivation":"Memory systems allow agents to retain and reuse information from past interactions, but they can also let malicious content persist.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27080","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_layerrag-bench_a4840c44","familyId":"bmf_046f75fe617f","name":"LayerRAG-Bench","oneLine":"Evaluates cross-layer reliability of agentic RAG systems on 240 tasks across 8 enterprise domains, with 9 fault scenarios and 2 contract modes; measures success at evidence, tool-contract, authorization, and session-state layers.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27353","pdf":"https://arxiv.org/pdf/2607.27353","project":null,"code":"https://github.com/MusaShams/layerrag-bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.27353"},"evidence":{"snippet":"We introduce LayerRAG-Bench, a controlled cross-layer reliability benchmark with 8 enterprise domains, 240 tasks, 9 fault scenarios, 2 contract modes, and 38,880 live task-level records across nine models from OpenAI, Anthropic, and Gemini.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27353"},"ranking":{"30d":{"score":50,"rank":76,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":285,"coverage":0.55,"confidence":"Low"}},"description":"Evaluates cross-layer reliability of agentic RAG systems on 240 tasks across 8 enterprise domains, with 9 fault scenarios and 2 contract modes; measures success at evidence, tool-contract, authorization, and session-state layers.","whyItMatters":"Groundedness alone misses operational failures. This benchmark isolates which layer a mitigation repairs, supporting targeted reliability improvements rather than blanket fixes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9b981f5fecedc0de8814da2481491e65ecb24ccf2d66b04126e0e0d6aada9049"},"motivation":"Agentic retrieval-augmented generation systems can produce answers that appear grounded while failing at the evidence, tool-contract, authorization, or session-state layer.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27353","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Musa Shams","organizationType":"community","sourceUrl":"https://github.com/MusaShams/layerrag-bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_hof-bench_be2a2445","familyId":"bmf_69c4fa77605b","name":"HoF-Bench","oneLine":"Evaluates vulnerability discovery in source code using 95 real CVEs across 8 repositories pinned at vulnerable commits, with a detector-blinded judge crediting findings that match code path, root cause, attack condition, and impact.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.27030","pdf":"https://arxiv.org/pdf/2607.27030","project":null,"code":"https://github.com/weareaisle/HoF-Bench","data":"https://huggingface.co/datasets/aisleinc/HoF-Bench","hfPaper":"https://huggingface.co/papers/2607.27030"},"evidence":{"snippet":"We introduce HoF-Bench (named after AISLE's public Hall of Fame), a benchmark built from 95 of these public AI-discovered CVEs across eight repositories pinned at vulnerable commits.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":272,"hfDatasetLikes":5},"source":{"type":"arxiv","id":"2607.27030"},"ranking":{"30d":{"score":67,"rank":35,"coverage":0.7,"confidence":"Medium","datasetDownloadRank":7,"datasetRankPopulation":20},"90d":{"score":57,"rank":152,"coverage":0.85,"confidence":"High","datasetDownloadRank":20,"datasetRankPopulation":51}},"description":"Evaluates vulnerability discovery in source code using 95 real CVEs across 8 repositories pinned at vulnerable commits, with a detector-blinded judge crediting findings that match code path, root cause, attack condition, and impact.","whyItMatters":"Provides a realistic test bed for comparing vulnerability scanners on rediscovery of known real-world vulnerabilities, with a strict scoring protocol and reusable dataset.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"806022f73a2c918564f8b111f9fdc1026a65bd72f6c8256ba6bd81f2b84b4854"},"motivation":"LLM-based analyzers have begun finding real vulnerabilities in mature open-source projects: AISLE's analyzer is credited with more than 280 CVEs across 78 projects, including OpenSSL, curl, and GnuTLS.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27030","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AISLE","organizationType":"company-research-lab","sourceUrl":"https://github.com/weareaisle/HoF-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_forgetbench_6c7f4d25","familyId":"bmf_6b9c23bcfd72","name":"ForgetBench","oneLine":"Proposes a benchmark for evaluating forgetting dynamics in language models under continual knowledge editing, with concept-based and scenario-based QA, but no public artifacts are available.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.26455","pdf":"https://arxiv.org/pdf/2607.26455","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.26455"},"evidence":{"snippet":"In this work, we propose ForgetBench, a benchmark designed to systematically characterize forgetting behavior in LLMs under continual knowledge editing.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26455"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Proposes a benchmark for evaluating forgetting dynamics in language models under continual knowledge editing, with concept-based and scenario-based QA, but no public artifacts are available.","whyItMatters":"Addresses knowledge retention over time, an important aspect for model updates.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"04aed9ccdd90b360d0698fc8f20e9c70b5f05db1bd14bffa807eee6cb317f187"},"motivation":"Large language models (LLMs) have demonstrated strong capabilities in knowledge acquisition and reasoning, yet their ability to retain previously acquired knowledge under repeated updates remains insufficiently understood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26455","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_explainbench_0a1efae5","familyId":"bmf_92c6f529f527","name":"ExplainBench","oneLine":"Evaluates code explanations from agents by checking whether explanations enable an LLM to correctly answer questions about intended behavior and patch effects, using a question-based suite derived from agent patches.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.26451","pdf":"https://arxiv.org/pdf/2607.26451","project":null,"code":"https://github.com/explainbench/explainbench-cli","data":null,"hfPaper":"https://huggingface.co/papers/2607.26451"},"evidence":{"snippet":"To bridge this gap, we propose ExplainBench, a benchmark to automatically evaluate explanations from coding agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":13,"hfDailySubmittedAt":"2026-08-05T00:00:00.000Z","githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26451"},"ranking":{"30d":{"score":41,"rank":88,"coverage":0.85,"confidence":"High"},"90d":{"score":33,"rank":309,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates code explanations from agents by checking whether explanations enable an LLM to correctly answer questions about intended behavior and patch effects, using a question-based suite derived from agent patches.","whyItMatters":"Agent explanations are often untrusted; this benchmark provides a quantitative way to compare explanation quality across agents, which is not captured by existing code-generation benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5fe6f8a58f7fab3118735bf4061feeb81ed2645e1b3f97274ebc178c70a356bb"},"motivation":"Large Language Model (LLM) agents have seen rapid adoption in software engineering.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26451","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_egosafe-bench_9e855ed0","familyId":"bmf_5dba8ef98ef1","name":"EgoSafe-Bench","oneLine":"EgoSafe-Bench evaluates visual safety understanding in first-person video, using 12,000 QA samples from 3,000 clips under the Hierarchical Reasoning Evaluation (HRE) protocol, which requires reasoning from feature anchoring to intent inference.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Safety","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.26518","pdf":"https://arxiv.org/pdf/2607.26518","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.26518"},"evidence":{"snippet":"To address this, we introduce EgoSafe-Bench, a benchmark specifically designed to probe forensic reasoning in egocentric safety scenarios.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26518"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"EgoSafe-Bench evaluates visual safety understanding in first-person video, using 12,000 QA samples from 3,000 clips under the Hierarchical Reasoning Evaluation (HRE) protocol, which requires reasoning from feature anchoring to intent inference.","whyItMatters":"Existing safety benchmarks rely on third-person footage and binary metrics, missing the causal reasoning gap in egocentric perception. EgoSafe-Bench provides a reusable protocol to assess whether LVLMs can move beyond correlation to forensic logic, informing model selection for safety-critical applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d4b06f9e2989d606d8b5fdfd974e1cc00cade31017738af31986054bd065a625"},"motivation":"Reliable visual safety understanding in real-world scenarios demands more than just object recognition; it requires causal reasoning under epistemic uncertainty.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26518","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_ecg-interpbench_df0017be","familyId":"bmf_d1b9cc173fbe","name":"ECG-InterpBench","oneLine":"ECG-InterpBench evaluates interpretability of ECG foundation models using sparse autoencoders with matched capacity, measuring reconstruction fidelity, clinical feature accessibility, and cross-seed reproducibility across 450 cells.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.27404","pdf":"https://arxiv.org/pdf/2607.27404","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.27404"},"evidence":{"snippet":"We introduce ECG-InterpBench, a benchmark designed to systematically evaluate the interpretability of ECG foundation-model representations.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27404"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ECG-InterpBench evaluates interpretability of ECG foundation models using sparse autoencoders with matched capacity, measuring reconstruction fidelity, clinical feature accessibility, and cross-seed reproducibility across 450 cells.","whyItMatters":"Performance-focused ECG benchmarks ignore whether representations are interpretable. This benchmark provides a controlled, reproducible framework for comparing models on interpretability, aiding clinical adoption where understanding model decisions is critical.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4a3801e94abcd9d28a5d01f9e8b61e4468f7f25e4274beccea887421e3dac8f7"},"motivation":"Existing benchmarks for electrocardiogram foundation models primarily evaluate downstream predictive performance, providing limited insight into whether their internal representations can be faithfully decomposed, clinically interpreted, or reproduced across independent analyses.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27404","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_ec-reason-bench_6994eec4","familyId":"bmf_e8fbcd70b60f","name":"EC-Reason-Bench","oneLine":"EC-Reason-Bench is a training-free diagnostic protocol for LLM enzyme classification, isolating four reasoning levers: output structure, external knowledge, reasoning structure, and robustness, to measure why LLMs fail on EC number prediction.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning","Factuality"],"topics":["Reasoning"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.26397","pdf":"https://arxiv.org/pdf/2607.26397","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.26397"},"evidence":{"snippet":"We propose EC-Reason-Bench, a training-free, diagnostic evaluation protocol built to answer two questions: why general LLMs score close to nothing on EC number prediction, and how much of that loss can be recovered without updating a single weight.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26397"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EC-Reason-Bench is a training-free diagnostic protocol for LLM enzyme classification, isolating four reasoning levers: output structure, external knowledge, reasoning structure, and robustness, to measure why LLMs fail on EC number prediction.","whyItMatters":"The protocol reveals that external knowledge is decisive and that reasoning acts as an arbiter among conflicting neighbors, showing that single-number leaderboards obscure the source of accuracy loss. This informs how to design knowledge-integration methods for protein function prediction.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"60aac0fe5509c09669f763909ee7d0f84bd843e63f84d7e5e3df9a9f64038552"},"motivation":"Enzyme function prediction is a hierarchical, knowledge-intensive form of protein function classification.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26397","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_agents4d_33599565","familyId":"bmf_0ea9b8cc532e","name":"AgentS4D","oneLine":"AgentS4D evaluates runtime safety of LLM-based workspace agents across a four-dimensional framework, with 328 risk-injected cases and seven lifecycle checkpoints, measuring unsafe behavior and evidence across six risk-entry sources and nine harms.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27294","pdf":"https://arxiv.org/pdf/2607.27294","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.27294"},"evidence":{"snippet":"We introduce AgentS4D, a sandboxed benchmark for lifecycle-wide runtime safety evaluation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27294"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AgentS4D evaluates runtime safety of LLM-based workspace agents across a four-dimensional framework, with 328 risk-injected cases and seven lifecycle checkpoints, measuring unsafe behavior and evidence across six risk-entry sources and nine harms.","whyItMatters":"Existing safety benchmarks focus on endpoints, missing risks that emerge during execution. This benchmark provides a structured way to assess agent safety across the lifecycle, showing that task completion does not imply safety and that testing one risk form can miss vulnerabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d2931d4ef15987426df2a5fd3166b316d072f59bbad4c5d4aa4a9a79e91534bc"},"motivation":"Large language model (LLM)-based workspace agents execute stateful, multi-step workflows across heterogeneous resources, external tools, and persistent state.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27294","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_worksurface-bench_7866ac7c","familyId":"bmf_639fa8c3c183","name":"WorkSurface-Bench","oneLine":"WorkSurface-Bench evaluates enterprise agents on knowledge routing across documents, tables, and graphs. It includes 1,151 atomic tasks with auditable reference answers and scoring for route, evidence, answer, and efficiency.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25765","pdf":"https://arxiv.org/pdf/2607.25765","project":null,"code":"https://github.com/haolpku/WorkSurface-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.25765"},"evidence":{"snippet":"We introduce WorkSurface-Bench, a benchmark for evaluating this capability as surface routing.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25765"},"ranking":{"30d":{"score":53,"rank":60,"coverage":0.85,"confidence":"High"},"90d":{"score":47,"rank":214,"coverage":0.7,"confidence":"Medium"}},"description":"WorkSurface-Bench evaluates enterprise agents on knowledge routing across documents, tables, and graphs. It includes 1,151 atomic tasks with auditable reference answers and scoring for route, evidence, answer, and efficiency.","whyItMatters":"It isolates surface routing from evidence acquisition and answer generation, showing that correct routing is necessary but insufficient. This helps diagnose why agents fail on multi-surface tasks and informs design of routing-aware systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1472e13f84f8ad0f4c160895993807ed362d13fb6b151507225ad3945576d897"},"motivation":"Enterprise agents often need to integrate heterogeneous knowledge sources: documents for narrative facts, tables for computation, and dependency graphs for file relationships.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25765","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"haolpku","organizationType":"academic-lab","sourceUrl":"https://github.com/haolpku/WorkSurface-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_stealthbench_8fce6971","familyId":"bmf_94101338dd62","name":"StealthBench","oneLine":"StealthBench measures operational stealth of autonomous offensive-security agents across six OPSEC dimensions, using 14 dockerized task scenarios and a three-model judge panel, with metrics like safe success rate and Stealth@Solve.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.26314","pdf":"https://arxiv.org/pdf/2607.26314","project":"https://stealthbench.com","code":"https://github.com/GangGreenTemperTatum/stealthbench","data":"https://huggingface.co/datasets/0xmoose/stealthbench","hfPaper":"https://huggingface.co/papers/2607.26314"},"evidence":{"snippet":"We introduce StealthBench,a benchmark that measures operational stealth in autonomous offensive-security agents across six operational security (OPSEC) dimensions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":5,"hfDailySubmittedAt":"2026-07-30T00:00:00.000Z","githubStars":7,"githubScope":"benchmark_repo","hfDatasetDownloads":35,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2607.26314"},"ranking":{"30d":{"score":61,"rank":44,"coverage":1.0,"confidence":"High","datasetDownloadRank":18,"datasetRankPopulation":20},"90d":{"score":45,"rank":234,"coverage":1.0,"confidence":"High","datasetDownloadRank":47,"datasetRankPopulation":51}},"description":"StealthBench measures operational stealth of autonomous offensive-security agents across six OPSEC dimensions, using 14 dockerized task scenarios and a three-model judge panel, with metrics like safe success rate and Stealth@Solve.","whyItMatters":"It addresses the gap where agents find vulnerabilities but fail tradecraft, systematic across models. The benchmark supports development of stealth-aware agents and automated OPSEC monitoring for safe deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f6e481bfa5ae085eb48f1356775981d044114bdd028b8c553f7f5c4ae84fdc04"},"motivation":"Stealth, the discipline of achieving an objective without revealing your presence, capabilities, or collected intelligence, is what separates sophisticated operators from detectable ones.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26314","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"stealthbench.com","organizationType":"benchmark-organization","sourceUrl":"https://stealthbench.com","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_secdrift_836e6631","familyId":"bmf_10522f747d5f","name":"SecDrift","oneLine":"SecDrift is a framework measuring sector-conditioned security drift in AI-generated code, evaluating vulnerability rates from LLMs when prompted with industry contexts versus neutral baselines across CISA sectors and CWE categories.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25225","pdf":"https://arxiv.org/pdf/2607.25225","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.25225"},"evidence":{"snippet":"We present SecDrift, a benchmark measuring sector-conditioned security drift: the change in static-analysis vulnerability rates when prompts are conditioned on industry contexts versus neutral baselines.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25225"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SecDrift is a framework measuring sector-conditioned security drift in AI-generated code, evaluating vulnerability rates from LLMs when prompted with industry contexts versus neutral baselines across CISA sectors and CWE categories.","whyItMatters":"It addresses whether domain-specific prompting affects code security, with findings that model choice matters more than prompt framing. The framework could guide deployment decisions, but its standalone benchmark status is unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ac53e40e73e6cff85237b5f26815fd1b3dd70242a7655bfd65c06c2981f802a3"},"motivation":"LLMs are increasingly used for code generation in critical infrastructure, yet the security effect of domain-specific prompting is understudied.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25225","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_rsibench-data_1ace1cc0","familyId":"bmf_e3268eaf0064","name":"RSIBench-Data","oneLine":"RSIBench-Data evaluates LLM agents as data-centric researchers, where agents iteratively revise training-data strategies for a fixed target model on six benchmarks, with real training and evaluation runs.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Self-Evolution"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25886","pdf":"https://arxiv.org/pdf/2607.25886","project":null,"code":"https://github.com/evolvent-ai/RSIBench-Data","data":null,"hfPaper":"https://huggingface.co/papers/2607.25886"},"evidence":{"snippet":"We introduce RSIBench-Data, a controlled benchmark of LLM agents as data-centric researchers with a fixed post-training stack.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":133,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25886"},"ranking":{"30d":{"score":77,"rank":14,"coverage":0.55,"confidence":"Low"},"90d":{"score":75,"rank":45,"coverage":0.55,"confidence":"Low"}},"description":"RSIBench-Data evaluates LLM agents as data-centric researchers, where agents iteratively revise training-data strategies for a fixed target model on six benchmarks, with real training and evaluation runs.","whyItMatters":"It isolates research capability from engineering, showing that current agents can improve from feedback but inconsistently. This provides an auditable testbed for capabilities needed in recursive self-improvement.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8433842326a170ba278916ef9854aff1083d43680966b7a32be6b16c16d1ee6c"},"motivation":"Recursive self-improvement requires turning evidence of model failures into better models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25886","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"evolvent-ai","organizationType":"company-research-lab","sourceUrl":"https://github.com/evolvent-ai/RSIBench-Data","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_reporeasoner_6d64b1f4","familyId":"bmf_edc74b50c095","name":"RepoReasoner","oneLine":"Evaluates long-context LLMs on repository-level code reasoning through Output Prediction and Call Chain Prediction tasks, using dynamic tracing and I/O rewriting to reduce memorization.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Reasoning"],"topics":["Long Context","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25996","pdf":"https://arxiv.org/pdf/2607.25996","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.25996"},"evidence":{"snippet":"We introduce RepoReasoner, a benchmark for evaluating repository-level code reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25996"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates long-context LLMs on repository-level code reasoning through Output Prediction and Call Chain Prediction tasks, using dynamic tracing and I/O rewriting to reduce memorization.","whyItMatters":"Assesses cross-file reasoning capabilities that are critical for real-world software engineering, identifying limitations beyond function-level benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"9e134949cc2337612f71aabfaf0e36c56bd9e5aacd353607585221c1faa5fbd8"},"motivation":"Recent large language models (LLMs) have shown strong performance on software engineering tasks, yet most existing benchmarks evaluate code reasoning at the function level, where all relevant information is localized.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25996","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"RepoReasoner Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.25996","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering","Long Context & Memory"],"domainScope":"general"},{"id":"bm_rdvsv2_d98f1ace","familyId":"bmf_57f7f6186a02","name":"RDVSv2","oneLine":"RDVSv2 is a large-scale benchmark for RGB-D video salient object detection, containing 249 video sequences with 29,077 annotated frames. It includes depth maps, optical flow, and eye-tracking-guided salient object masks. The benchmark provides a fixed dataset and evaluation protocol for comparing models on this task.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25392","pdf":"https://arxiv.org/pdf/2607.25392","project":null,"code":"https://github.com/ltynick/RDVSv2","data":null,"hfPaper":"https://huggingface.co/papers/2607.25392"},"evidence":{"snippet":"We introduce RDVSv2, a large-scale benchmark for RGB-D video salient object detection (RGB-D VSOD) with dense frame-level annotations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25392"},"ranking":{"30d":{"score":61,"rank":43,"coverage":0.55,"confidence":"Low"},"90d":{"score":49,"rank":203,"coverage":0.55,"confidence":"Low"}},"description":"RDVSv2 is a large-scale benchmark for RGB-D video salient object detection, containing 249 video sequences with 29,077 annotated frames. It includes depth maps, optical flow, and eye-tracking-guided salient object masks. The benchmark provides a fixed dataset and evaluation protocol for comparing models on this task.","whyItMatters":"Existing RGB-D VSOD datasets are limited in scale and annotation quality, hindering progress. RDVSv2 offers a larger, more diverse, and challenging benchmark, enabling more robust evaluation and comparison of models, and supporting the development of methods that can handle real-world scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"64e9f1b0c825d0ba6a0fdb39fb41668e0465a2950a36f3a4309d7e26da23db98"},"motivation":"We introduce RDVSv2, a large-scale benchmark for RGB-D video salient object detection (RGB-D VSOD) with dense frame-level annotations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ACMMM 2026","evidence":"Accepted to ACMMM 2026","evidenceUrl":"https://arxiv.org/abs/2607.25392","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ACMMM 2026","reviewStatus":"accepted","decisionRaw":"Accepted to ACMMM 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.25392","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to ACMMM 2026","level":"author-claim"}]}],"publishers":[{"name":"RDVSv2 team","organizationType":"academic-lab","sourceUrl":"https://github.com/ltynick/RDVSv2","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_polistemics_ea35d629","familyId":"bmf_14debe1298ea","name":"Polistemics","oneLine":"Polistemics evaluates LLMs as mediators of political information across controlled settings varying evidence clarity, noise, and consistency, using a diagnostic benchmark grounded in Epistemic Modesty.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25953","pdf":"https://arxiv.org/pdf/2607.25953","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.25953"},"evidence":{"snippet":"We introduce Polistemics, a theory-grounded diagnostic benchmark for evaluating LLMs as mediators of political information in elections.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25953"},"ranking":{"30d":{"score":47,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Polistemics evaluates LLMs as mediators of political information across controlled settings varying evidence clarity, noise, and consistency, using a diagnostic benchmark grounded in Epistemic Modesty.","whyItMatters":"High aggregate scores can mask systematic failures in LLM political mediation, particularly under ambiguous or contradictory evidence, affecting citizens' ability to make informed electoral decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7aa2097eb092a875d1e16ec5079cbce1399c3380f441a114c6872db772b2e86b"},"motivation":"As LLMs increasingly shape the political information citizens rely on, no standard exists to assess whether they do so responsibly.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25953","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_orchbench_2f9fbf5c","familyId":"bmf_11407a7c2e35","name":"OrchBench","oneLine":"OrchBench evaluates multi-agent orchestration plans in isolation using deterministic simulation. It constructs DAGs from real-world tasks and scores plans on result quality, makespan, and token cost without executing worker agents.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25656","pdf":"https://arxiv.org/pdf/2607.25656","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.25656"},"evidence":{"snippet":"We present OrchBench, a simulation-based benchmark for evaluating multi-agent orchestration plans in isolation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25656"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OrchBench evaluates multi-agent orchestration plans in isolation using deterministic simulation. It constructs DAGs from real-world tasks and scores plans on result quality, makespan, and token cost without executing worker agents.","whyItMatters":"OrchBench provides a fast, token-efficient evaluation of orchestration plans, decoupling planning quality from worker capabilities and environmental noise. Its simulated scores correlate strongly with real executions, enabling cost-effective comparison and diagnosis of multi-agent planners.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b87ecaf5ac12614286895adf6b43781cf1be0be29071f9ce08b2f4a9704ae3db"},"motivation":"Complex tasks often decompose into parallelizable yet interdependent subtasks, making orchestration critical to the performance of multi-agent systems (MAS).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25656","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_omniphys_eb0844b7","familyId":"bmf_1f087830f505","name":"OmniPhys","oneLine":"OmniPhys is a benchmark of 1,551 text-to-image generation samples grounded in a Physical Knowledge Graph, aligned with PhET simulations and curricula. It evaluates physical commonsense in generated images using a dual-path verification protocol.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.CV"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25641","pdf":"https://arxiv.org/pdf/2607.25641","project":null,"code":"https://github.com/zjukg/OmniPhys","data":null,"hfPaper":"https://huggingface.co/papers/2607.25641"},"evidence":{"snippet":"To address these challenges, we introduce OmniPhys, a rigorous benchmark of 1,551 samples grounded in a Physical Knowledge Graph.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25641"},"ranking":{"30d":{"score":34,"rank":135,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":358,"coverage":0.55,"confidence":"Low"}},"description":"OmniPhys is a benchmark of 1,551 text-to-image generation samples grounded in a Physical Knowledge Graph, aligned with PhET simulations and curricula. It evaluates physical commonsense in generated images using a dual-path verification protocol.","whyItMatters":"Existing benchmarks use coarse descriptions and fail to diagnose specific physical principles. OmniPhys provides a fine-grained, curriculum-aligned evaluation to identify systemic physical reasoning gaps in image generation models, supporting targeted improvement.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"97c19e30b9cdcf993a1194adeb7f5614e69344cc3e4a4a4ee494d460edd5a4c9"},"motivation":"While text-to-image models exhibit remarkable visual fidelity, they frequently violate fundamental physical commonsense.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"KDD 2026 DB track","evidence":"accepted by KDD 2026 DB track","evidenceUrl":"https://arxiv.org/abs/2607.25641","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"KDD 2026 DB track","reviewStatus":"accepted","decisionRaw":"accepted by KDD 2026 DB track","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.25641","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"accepted by KDD 2026 DB track","level":"author-claim"}]}],"publishers":[{"name":"ZJUKG","organizationType":"academic-lab","sourceUrl":"https://github.com/zjukg/OmniPhys","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_neurai-vn-benchmark_10fca573","familyId":"bmf_5f53938b8ce8","name":"Neurai-VN Benchmark","oneLine":"Evaluates machine learning models on the Neurai-VN dataset for mental health classification. Four binary tasks (healthy control vs. depression, anxiety, clinical, and depression vs. anxiety) are defined using subject-wise cross-validation and standardized feature groups. Baseline models include linear, tree-based, and neural networks, with mean F1 scores reported.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25232","pdf":"https://arxiv.org/pdf/2607.25232","project":null,"code":"https://github.com/neurai-vn/Neurai-VN-benchmark","data":null,"hfPaper":"https://huggingface.co/papers/2607.25232"},"evidence":{"snippet":"In this work, we introduce a reproducible machine learning benchmark using the Neurai-VN dataset, a multimodal digital phenotyping dataset collected from 100 Vietnamese adults over two weeks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25232"},"ranking":{"30d":{"score":56,"rank":54,"coverage":0.55,"confidence":"Low"},"90d":{"score":44,"rank":244,"coverage":0.55,"confidence":"Low"}},"description":"Evaluates machine learning models on the Neurai-VN dataset for mental health classification. Four binary tasks (healthy control vs. depression, anxiety, clinical, and depression vs. anxiety) are defined using subject-wise cross-validation and standardized feature groups. Baseline models include linear, tree-based, and neural networks, with mean F1 scores reported.","whyItMatters":"Provides a standardized evaluation protocol for multimodal digital phenotyping in mental health, addressing inconsistencies in preprocessing and evaluation across datasets. Offers comparable baselines for future research.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f6fb97748d48ddc3666de11ab838b1f6252ec4d1d47ca8b2e9830cacdda2d1a6"},"motivation":"Digital phenotyping (DP) using smartphones and wearable devices has emerged as a promising approach for assessing mental health, particularly depression and anxiety.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25232","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Neurai-VN","organizationType":"community","sourceUrl":"https://github.com/neurai-vn/Neurai-VN-benchmark","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_medit-bench_8ae40735","familyId":"bmf_97c8c854879f","name":"MEDit-Bench","oneLine":"Evaluates message-driven narrative video editing with long-form videos paired with multiple editing messages and multiple professional edits per message, using temporal alignment metrics and additional annotations for message ambiguity and contextfulness.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25300","pdf":"https://arxiv.org/pdf/2607.25300","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.25300"},"evidence":{"snippet":"For evaluating message-driven video editing, we present \\textbf{MEDit-Bench}, a dataset and benchmark, which pairs long-form videos with multiple editing messages and multiple professionally produced edits per message, demonstrating that different messages yield substantially different edits from the same source.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25300"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates message-driven narrative video editing with long-form videos paired with multiple editing messages and multiple professional edits per message, using temporal alignment metrics and additional annotations for message ambiguity and contextfulness.","whyItMatters":"Addresses the gap in video editing evaluation by accounting for diverse editorial intents, providing a protocol to compare model and human performance on narrative-driven editing, and offering stratification by message difficulty.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6982c6768baa84a2a6ece02f27e26e54ce7fb6bc2895e5bb726ee658676c557a"},"motivation":"Video editing is fundamentally message-driven: even from the same source footage, the selected shots change depending on the narrative the editor wishes to convey.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25300","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ih-benchmark_f309c884","familyId":"bmf_505ebf5f9c8e","name":"IH-Benchmark","oneLine":"IH-Benchmark evaluates instruction-hierarchy robustness in LLMs via conflicting instructions from system, user, and tool outputs, covering 44 constraint families across five domains with a binary pass/fail protocol.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Robustness"],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.25987","pdf":"https://arxiv.org/pdf/2607.25987","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.25987"},"evidence":{"snippet":"We present IH-Benchmark, a conflict-centered benchmark for instruction-hierarchy robustness across direct system-user conflicts (S>U) and tool-mediated user-tool (U>T) conflicts.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25987"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"IH-Benchmark evaluates instruction-hierarchy robustness in LLMs via conflicting instructions from system, user, and tool outputs, covering 44 constraint families across five domains with a binary pass/fail protocol.","whyItMatters":"Existing instruction-hierarchy benchmarks cover limited conflict types and tool interactions. IH-Benchmark provides a systematic evaluation across conflict surfaces, constraint types, and attack presentations, revealing that robustness is not a single capability but a set of behaviors with distinct failure modes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1ed86f357979990217f303f2c9d031227465185497921e25e38937b00e5da8f6"},"motivation":"When a language model receives conflicting instructions from different priority levels, which one does it actually follow?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25987","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_desktop-delta-bench_ac655444","familyId":"bmf_40d33a605fa0","name":"Desktop-Delta Bench","oneLine":"Desktop-Delta Bench (DDB) evaluates computer-use models on step-level GUI transition understanding through two tasks: temporal ordering of 3-frame observations and before-after pair classification with five action types, covering 2,013 human-verified instances across ~15 applications.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.26041","pdf":"https://arxiv.org/pdf/2607.26041","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.26041"},"evidence":{"snippet":"We introduce Desktop-Delta Bench (DDB), an offline step-level benchmark with 2,013 human-verified instances from novel, multi-app Linux trajectories across ~15 applications and 50 task domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26041"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Desktop-Delta Bench (DDB) evaluates computer-use models on step-level GUI transition understanding through two tasks: temporal ordering of 3-frame observations and before-after pair classification with five action types, covering 2,013 human-verified instances across ~15 applications.","whyItMatters":"Existing benchmarks focus on end-task success or single-frame grounding, missing the ability to reconstruct causal transitions. DDB provides a diagnostic layer for state verification, source tracking, and context-aware control, enabling targeted improvements in desktop CUA reliability and recovery.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"48ef5f046f9fa5edccd3ad09ee29b94020f44722690242ad66d15d01f536730a"},"motivation":"Computer-use agents (CUAs) increasingly act through desktop GUIs to complete long-horizon tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26041","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_debtbench_7013b5c5","familyId":"bmf_89860188c215","name":"DebtBench","oneLine":"DebtBench evaluates negotiation dialogue systems in debt collection with persona-enriched user profiles to reflect behavioral heterogeneity. It uses a dataset of negotiation scenarios with multiple user personas, scoring financial recovery and interaction experience metrics.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25218","pdf":"https://arxiv.org/pdf/2607.25218","project":null,"code":"https://github.com/YYuHhhh/DebtNegotiation","data":null,"hfPaper":"https://huggingface.co/papers/2607.25218"},"evidence":{"snippet":"To bridge this gap, we propose DebtBench, the first public persona-enriched debt collection benchmark, that highlights behavioral heterogeneity in negotiation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25218"},"ranking":{"30d":{"score":34,"rank":136,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":359,"coverage":0.55,"confidence":"Low"}},"description":"DebtBench evaluates negotiation dialogue systems in debt collection with persona-enriched user profiles to reflect behavioral heterogeneity. It uses a dataset of negotiation scenarios with multiple user personas, scoring financial recovery and interaction experience metrics.","whyItMatters":"Existing negotiation benchmarks assume static, rational users, missing real-world behavioral heterogeneity. DebtBench provides a more realistic testbed for financial negotiation, allowing evaluation of models' ability to handle diverse user behaviors and optimize both recovery and user experience.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a048cfa23c639a5575f2662eec4b8c8b210f934092fcaa0797a92ee1d36b2291"},"motivation":"Debt collection is a critical negotiation task in the financial industry, with strong practical relevance and exceptional academic value as a behaviorally rich, high-stakes testbed for human-centered dialogue systems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25218","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DebtBench Team","organizationType":"academic-lab","sourceUrl":"https://github.com/YYuHhhh/DebtNegotiation","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_clinlens_3ca0c7ce","familyId":"bmf_0887f3ebbae5","name":"ClinLens","oneLine":"ClinLens evaluates clinical data-science agents on 200 executable tasks over five linked MIMIC resources (EHR, notes, ECG, chest X-rays, echocardiograms), organized by a 4x5 taxonomy of patient-time scopes and analysis capabilities. Scoring uses a STRICTPASS metric requiring correct artifacts, semantics, and final answers.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.26155","pdf":"https://arxiv.org/pdf/2607.26155","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.26155"},"evidence":{"snippet":"We introduce CLINLENS, a benchmark of 200 executable tasks over five linked MIMIC resources spanning structured electronic health records, notes, electrocardiograms, chest radiographs, and echocardiograms.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26155"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ClinLens evaluates clinical data-science agents on 200 executable tasks over five linked MIMIC resources (EHR, notes, ECG, chest X-rays, echocardiograms), organized by a 4x5 taxonomy of patient-time scopes and analysis capabilities. Scoring uses a STRICTPASS metric requiring correct artifacts, semantics, and final answers.","whyItMatters":"Existing benchmarks isolate medical QA or table reasoning, lacking integrated longitudinal clinical data science. ClinLens fills this gap with a program-first reverse synthesis approach, exposing a gap between runnable code and correct clinical analysis, guiding improvements in clinical agent reliability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2514840c4290af7724ea0fced91dd766511f6a845a9ed87a6edd46736e690d6f"},"motivation":"Clinical data-science agents must transform heterogeneous longitudinal records into auditable analyses, yet existing benchmarks largely isolate medical question answering, structured-table reasoning, or generic scientific repositories.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26155","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_clbench-v_d435f082","familyId":"bmf_b6a0b47faa26","name":"CLBench-V","oneLine":"CLBench-V evaluates multimodal context learning across three dimensions: context grounding, new information application, and new knowledge learning. It includes 3,443 instances across 14 subdatasets spanning science, finance, long-document understanding, spatial reasoning, and web-based VQA.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25294","pdf":"https://arxiv.org/pdf/2607.25294","project":null,"code":"https://github.com/IamLihua/CLBench-V","data":null,"hfPaper":"https://huggingface.co/papers/2607.25294"},"evidence":{"snippet":"We introduce CLBench-V, a benchmark for multimodal context learning that addresses the difficulty of localizing where context use breaks down by organizing tasks around three dimensions: context grounding, new information application, and new knowledge learning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":49,"hfDailySubmittedAt":"2026-07-30T00:00:00.000Z","githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25294"},"ranking":{"30d":{"score":73,"rank":24,"coverage":0.85,"confidence":"High"},"90d":{"score":53,"rank":179,"coverage":0.7,"confidence":"Medium"}},"description":"CLBench-V evaluates multimodal context learning across three dimensions: context grounding, new information application, and new knowledge learning. It includes 3,443 instances across 14 subdatasets spanning science, finance, long-document understanding, spatial reasoning, and web-based VQA.","whyItMatters":"Existing context learning benchmarks focus on text, missing multimodal settings where context is in figures, tables, and maps. CLBench-V provides a structured evaluation to localize where context use breaks down, aiding progress in multimodal models for real-world tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"43a1a78b46d23c2eb8796c5c81c281a3cfd93d5391e8d028810454aea4ef2645"},"motivation":"Real-world tasks often require models to learn from task-specific context rather than relying only on pre-trained knowledge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25294","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"CLBench-V Team","organizationType":"academic-lab","sourceUrl":"https://github.com/IamLihua/CLBench-V","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_bg-real_ba34eeb5","familyId":"bmf_6a40ec51857c","name":"BG-REAL","oneLine":"BG-REAL is a benchmark for background manipulation detection and localization in images. It contains 7,000 processed samples (6,000 public-data anchored, 1,000 synthetic) over six edit families with matched authentic controls, source-group splits, and quality control.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.26232","pdf":"https://arxiv.org/pdf/2607.26232","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.26232"},"evidence":{"snippet":"We introduce BG-REAL, a public real-data anchored benchmark package for background manipulation detection and localization.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26232"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"BG-REAL is a benchmark for background manipulation detection and localization in images. It contains 7,000 processed samples (6,000 public-data anchored, 1,000 synthetic) over six edit families with matched authentic controls, source-group splits, and quality control.","whyItMatters":"Existing image forensics benchmarks focus on object-centric manipulations, missing background edits. BG-REAL provides a targeted evaluation with matched controls to measure false-positive rates from re-encoding artifacts, exposing a shared shortcut risk across baselines.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"95c49a9214a32e9c3fd941275aa030eb8a7c6ddf2a5eac0d3877fc40bf785bc1"},"motivation":"Background manipulation is a practical but under-specified image-forensics setting: the manipulated evidence can sit outside the salient foreground object, while many evaluations emphasize object-centric copy-move, splicing, or generic synthetic edits.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26232","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_annobench_f60658c7","familyId":"bmf_87eaf3c4ce78","name":"AnnoBench","oneLine":"AnnoBench evaluates visualization annotation generation across four representation formats, five chart description conditions, and two prompt specification levels. It uses a VLM-as-a-judge protocol aligned with human assessment to score annotation quality.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.HC"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25911","pdf":"https://arxiv.org/pdf/2607.25911","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.25911"},"evidence":{"snippet":"We introduce AnnoBench, a benchmark for visualization annotation that materializes the inherent challenges of this domain in a structured and testable manner.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.25911"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AnnoBench evaluates visualization annotation generation across four representation formats, five chart description conditions, and two prompt specification levels. It uses a VLM-as-a-judge protocol aligned with human assessment to score annotation quality.","whyItMatters":"No existing benchmark tests whether annotation tools meet visual, semantic, and stylistic constraints. AnnoBench provides a structured evaluation framework to advance annotation automation and visualization generation pipelines.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"beb8b31939b5b03669e6046dc49148a504fe11ca76f2a39891b6796ba17a9607"},"motivation":"Annotation is among the most demanding visualization tasks to automate, as it simultaneously requires correctly navigating visual, semantic, and stylistic constraints.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25911","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_ampbench-mt_ce75d23c","familyId":"bmf_d0c8104c4688","name":"AMPBench-MT","oneLine":"AMPBench-MT evaluates antimicrobial peptide prediction across binary recognition, species-conditioned potency regression, and endpoint-specific safety readouts (hemolysis, toxicity, selectivity) under a sequence-homology-controlled protocol. It includes 13 source databases and multiple task configurations.","area":"Safety & Trustworthiness","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.25518","pdf":"https://arxiv.org/pdf/2607.25518","project":null,"code":null,"data":"https://huggingface.co/datasets/ZihengZhou06/AMPBench-MT","hfPaper":"https://huggingface.co/papers/2607.25518"},"evidence":{"snippet":"To address this problem, we introduce AMPBench-MT, a provenance-preserving benchmark that standardizes canonical peptide records and organizes them into binary recognition, species-conditioned pMIC regression, and endpoint-specific potency and safety-facing readouts.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":307,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2607.25518"},"ranking":{"30d":{"score":53,"rank":58,"coverage":0.15,"confidence":"Low","datasetDownloadRank":6,"datasetRankPopulation":20},"90d":{"score":55,"rank":169,"coverage":0.3,"confidence":"Low","datasetDownloadRank":18,"datasetRankPopulation":51}},"description":"AMPBench-MT evaluates antimicrobial peptide prediction across binary recognition, species-conditioned potency regression, and endpoint-specific safety readouts (hemolysis, toxicity, selectivity) under a sequence-homology-controlled protocol. It includes 13 source databases and multiple task configurations.","whyItMatters":"Existing AMP benchmarks focus on binary recognition, but follow-up decisions need assay-derived evidence. AMPBench-MT provides a joint evaluation with homology-controlled splits to reveal that high binary performance does not guarantee assay-endpoint behavior.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"07fd4dc06b2bb0d6870849797c961b38c190b015b407d037f771ad32e82f1e03"},"motivation":"Computational AMP discovery is often evaluated through AMP/non-AMP recognition, yet follow-up decisions depend on assay-derived evidence such as target-species potency, hemolysis, toxicity, and selectivity.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.25518","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ZihengZhou06","organizationType":"community","sourceUrl":"https://huggingface.co/datasets/ZihengZhou06/AMPBench-MT","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_stanceflip_415aae19","familyId":"bmf_e22e85b4e626","name":"StanceFlip","oneLine":"StanceFlip is a benchmark for multimodal conversational stance flipping forecasting with two subtasks: sextuple extraction and flip attribution.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.24191","pdf":"https://arxiv.org/pdf/2607.24191","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.24191"},"evidence":{"snippet":"To address these limitations, we propose StanceFlip, a benchmark designed for multimodal conversational stance flipping forecasting over multi-turn dialogues across five modalities and multi-scenarios, which includes two novel subtasks: 1) Multimodal Stance Sextuple Extraction, extracting holder, target, emotion, sentiment, stance, and rationale as static state snapshots of dialogue to capture fine-grained cognitive structures.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24191"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"StanceFlip is a benchmark for multimodal conversational stance flipping forecasting with two subtasks: sextuple extraction and flip attribution.","whyItMatters":"It addresses gaps in dynamic stance evolution and multimodal cues, but lacks a public reuse path and stable scoring contract.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8e511dd5b77afe9d288726c2d0bd2353e829077cef005ae11e334becb7007a0e"},"motivation":"Conversational stance detection has shifted from static text analysis to dynamic multimodal modeling.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24191","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_routebench_6fe659bf","familyId":"bmf_16ecd82fb9ee","name":"ROUTEBENCH","oneLine":"ROUTEBENCH evaluates whether transformers can learn latent algorithm routing across four solver families (ridge, lasso, Huber, kNN) in a controlled diagnostic setting.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Robustness"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.24471","pdf":"https://arxiv.org/pdf/2607.24471","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.24471"},"evidence":{"snippet":"We introduce ROUTEBENCH, a diagnostic benchmark whose regimes differentially favor global shrinkage, sparsity, robustness, and locality, operationalized by ridge-like, lasso-like, Huber-like, and kNN-like family representatives.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24471"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ROUTEBENCH evaluates whether transformers can learn latent algorithm routing across four solver families (ridge, lasso, Huber, kNN) in a controlled diagnostic setting.","whyItMatters":"Provides controlled evidence on internal routing in transformers, but lacks a standalone public comparison path and is primarily a research probe.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f67a2cb06183b1d8085e2266d7c81b1f360e6a534ec56ce0b6f83ad1302ea8f2"},"motivation":"A central question in the in-context learning literature is whether transformers can organize episode-level adaptation around different inductive-bias families.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"COLM 2026","evidence":"Accepted by COLM 2026","evidenceUrl":"https://arxiv.org/abs/2607.24471","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"COLM 2026","reviewStatus":"accepted","decisionRaw":"Accepted by COLM 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.24471","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by COLM 2026","level":"author-claim"}]}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_perceptionbench_27ae03ae","familyId":"bmf_885ad8ebb2ad","name":"PerceptionBench","oneLine":"PerceptionBench evaluates atomic visual perception in MLLMs with 3,000 verified questions isolating ten perceptual capabilities, based on an error taxonomy from 42 benchmarks.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.24957","pdf":"https://arxiv.org/pdf/2607.24957","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.24957"},"evidence":{"snippet":"We introduce PerceptionBench, a benchmark specifically designed to evaluate the atomic visual perception capabilities of Multimodal Large Language Models (MLLMs).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":20,"hfDailySubmittedAt":"2026-07-29T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24957"},"ranking":{"30d":{"score":59,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":55,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"PerceptionBench evaluates atomic visual perception in MLLMs with 3,000 verified questions isolating ten perceptual capabilities, based on an error taxonomy from 42 benchmarks.","whyItMatters":"Addresses the need for a capability-level standard to diagnose visual perception boundaries, showing that current MLLMs remain below 60% accuracy.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1dbc8e54ea45881534d104682db2e2e19d0ded65ba923bbca03fd4741fd614d7"},"motivation":"We introduce PerceptionBench, a benchmark specifically designed to evaluate the atomic visual perception capabilities of Multimodal Large Language Models (MLLMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24957","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_lu-500_07127332","familyId":"bmf_71cc3bc652c4","name":"LU-500","oneLine":"LU-500 evaluates concept unlearning for logos with nearly 10,000 pairs, including explicit and implicit contextual tracks, and a multi-grained protocol measuring local removal and global preservation.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.24101","pdf":"https://arxiv.org/pdf/2607.24101","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.24101"},"evidence":{"snippet":"We introduce LU-500, a logo-unlearning benchmark built from Fortune Global 500 companies to study this localized and semantically entangled setting.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24101"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LU-500 evaluates concept unlearning for logos with nearly 10,000 pairs, including explicit and implicit contextual tracks, and a multi-grained protocol measuring local removal and global preservation.","whyItMatters":"Provides a specialized benchmark for a challenging unlearning scenario, enabling comparison of methods on localized and entangled visual concepts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"09a62db5a44d8e9be375ff396828b0ac449a5190dd8f42bb871865b7a7e59c04"},"motivation":"Concept unlearning is increasingly used to limit the reproduction of protected or unsafe visual concepts in text-to-image models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24101","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ins-actbench_3f617677","familyId":"bmf_362d5d3241c5","name":"INS-ActBench","oneLine":"INS-ActBench evaluates actuarial capability in LLMs across knowledge, case reasoning, and tool-based practice with 12,050 tasks from public exams.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.24273","pdf":"https://arxiv.org/pdf/2607.24273","project":null,"code":"https://github.com/FDU-INS/INS-ActBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.24273"},"evidence":{"snippet":"We introduce \\textbf{INS-ActBench}, a comprehensive benchmark for evaluating professional actuarial capability in LLMs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24273"},"ranking":{"30d":{"score":50,"rank":77,"coverage":0.55,"confidence":"Low"},"90d":{"score":39,"rank":286,"coverage":0.55,"confidence":"Low"}},"description":"INS-ActBench evaluates actuarial capability in LLMs across knowledge, case reasoning, and tool-based practice with 12,050 tasks from public exams.","whyItMatters":"Provides a reproducible foundation for assessing professional actuarial assistance, revealing capability gaps in case reasoning and tool use.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"de113beb623652cc29500c3ef9bc30ad7642c83909ce4dc60e633f626b77e794"},"motivation":"Large Language Models (LLMs) have shown strong potential in financial reasoning, but existing benchmarks often evaluate domain knowledge, numerical reasoning, long-context understanding, and tool use in separate settings.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24273","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FDU-INS","organizationType":"academic-lab","sourceUrl":"https://github.com/FDU-INS/INS-ActBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_graphrarebench_9ff01b1d","familyId":"bmf_1c6412c93a36","name":"GraphRareBench","oneLine":"GraphRareBench evaluates phenotype-driven rare-disease ranking with 2,365 cases, 18,093 target-confounder pairs, and auditable evidence records.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["q-bio.QM"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.24878","pdf":"https://arxiv.org/pdf/2607.24878","project":null,"code":"https://github.com/GUI0609/GraphRareBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.24878"},"evidence":{"snippet":"We introduce GraphRareBench, a provenance-preserving benchmark containing 2,365 ontology-derived cases and 18,093 target-confounder pairs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24878"},"ranking":{"30d":{"score":34,"rank":137,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":360,"coverage":0.55,"confidence":"Low"}},"description":"GraphRareBench evaluates phenotype-driven rare-disease ranking with 2,365 cases, 18,093 target-confounder pairs, and auditable evidence records.","whyItMatters":"Offers a transparent, evidence-aware evaluation for diagnostic systems, measuring retrieval and hard-confounder discrimination in a provenance-preserving setting.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"32cc2d0e21e8e20b1d07804d0e1e065b90a29de8c970df54bfd51fbd0cd76f05"},"motivation":"Phenotype-driven diagnostic benchmarks usually report the rank of the reference disease, but they rarely reveal which plausible alternatives are ranked above it or what evidence a tool-using model examines before making its decision.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24878","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GraphRareBench team","organizationType":"academic-lab","sourceUrl":"https://github.com/GUI0609/GraphRareBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_gauge_c5d48106","familyId":"bmf_a90fd9a9a1e6","name":"GAUGE","oneLine":"GAUGE evaluates agent-built financial valuation models against observed analyst practice using 56 facets, eight validity gates, and a failure-aware score over 196 tasks.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.24889","pdf":"https://arxiv.org/pdf/2607.24889","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.24889"},"evidence":{"snippet":"We introduce GAUGE, a benchmark for evaluating agent-built valuation models against observed analyst practice rather than a single point answer.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24889"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GAUGE evaluates agent-built financial valuation models against observed analyst practice using 56 facets, eight validity gates, and a failure-aware score over 196 tasks.","whyItMatters":"Provides a benchmark that avoids penalizing legitimate disagreement, enabling fair comparison of agents on financial modeling and judgment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"241f8c88b248e236fca5562ceb58f39ba684b023b452289e162279bcbbb90989"},"motivation":"Financial models combine public disclosures with analyst assumptions to produce forecasts and valuations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24889","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_filmbench_2191702d","familyId":"bmf_cf4912a2e3be","name":"FilmBench","oneLine":"FilmBench evaluates text-to-video and reference-to-video generation using a Cinematic Language taxonomy with 1,169 prompts and 35 sub-metrics.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.24241","pdf":"https://arxiv.org/pdf/2607.24241","project":null,"code":"https://github.com/Neo-yk/FilmOps","data":null,"hfPaper":"https://huggingface.co/papers/2607.24241"},"evidence":{"snippet":"We introduce FilmBench, a text-to-video (T2V) and reference-to-video (R2V) benchmark grounded in the professional Cinematic Language of the film- academy tradition and co-developed with directors and faculty from the Beijing Film Academy and the Hujing Digital Media & Entertainment Group film studio.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":6,"hfDailySubmittedAt":"2026-07-28T00:00:00.000Z","githubStars":27,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24241"},"ranking":{"30d":{"score":74,"rank":22,"coverage":0.85,"confidence":"High"},"90d":{"score":68,"rank":83,"coverage":0.7,"confidence":"Medium"}},"description":"FilmBench evaluates text-to-video and reference-to-video generation using a Cinematic Language taxonomy with 1,169 prompts and 35 sub-metrics.","whyItMatters":"Provides a film-grade benchmark with professional criteria, enabling assessment of cinematic craft beyond basic plausibility.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5823e705620fe38912c6c653d6404f6467cb393a028cf5277e9cd51b3844de10"},"motivation":"Progress in video generation keeps narrowing the visual gap between AI-generated and professionally produced footage, yet most benchmarks still draw prompts from web sources or LLM templates and score them with untrained, generic multimodal models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24241","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FilmBench team","organizationType":"academic-lab","sourceUrl":"https://github.com/Neo-yk/FilmOps","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_exe-bench_d4e86d18","familyId":"bmf_e34bb0462558","name":"EXE-Bench","oneLine":"EXE-Bench evaluates AI-based Windows malware detectors on performance, temporal robustness, adversarial robustness, and computational overhead, aggregated into a single score.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.24177","pdf":"https://arxiv.org/pdf/2607.24177","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.24177"},"evidence":{"snippet":"For these reasons, we develop EXE-Bench, a comprehensive benchmark of AI-based Windows malware detectors.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24177"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EXE-Bench evaluates AI-based Windows malware detectors on performance, temporal robustness, adversarial robustness, and computational overhead, aggregated into a single score.","whyItMatters":"Provides a comprehensive benchmark to guide deployment decisions, highlighting tradeoffs between feature-engineered and deep network detectors.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1fce9ab62c0e02381b7a39866eebc53cdedc67cc162a1d3c94932f8f1eeddb1b"},"motivation":"Due to the lack of systematic evaluations, we are not yet able to determine which AI-based Windows malware detector to deploy in production, since existing evaluations (i) differ in terms of data used for both training and testing; (ii) do not consider temporal analysis to showcase whether models withstand the passage of time; (iii) avoid security evaluations with adversarial attacks that could highlight their britt…","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24177","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_erunderstand_657f59fd","familyId":"bmf_961dfc017065","name":"ERUnderstand","oneLine":"ERUnderstand evaluates vision-language models on structured understanding of entity-relationship diagrams. The benchmark contains 2,960 diagrams across curated educational sources, real-world schemas, and synthetic data, with standardized machine-readable JSON annotations. Scoring uses F1, BLEU, and graph edit distance to measure how accurately models recover schema elements such as entities, relationships, attributes, and extended ER constructs.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.24707","pdf":"https://arxiv.org/pdf/2607.24707","project":null,"code":"https://github.com/salinaria/ERUnderstand","data":null,"hfPaper":"https://huggingface.co/papers/2607.24707"},"evidence":{"snippet":"We introduce ERUnderstand, the first large-scale benchmark for structured understanding of ER diagrams, comprising 2,960 diagrams collected from curated educational sources, real-world schemas, and synthetically generated examples spanning diverse domains, notations, complexity levels, and Extended Entity-Relationship (EER) constructs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24707"},"ranking":{"30d":{"score":60,"rank":45,"coverage":0.85,"confidence":"High"},"90d":{"score":50,"rank":193,"coverage":0.7,"confidence":"Medium"}},"description":"ERUnderstand evaluates vision-language models on structured understanding of entity-relationship diagrams. The benchmark contains 2,960 diagrams across curated educational sources, real-world schemas, and synthetic data, with standardized machine-readable JSON annotations. Scoring uses F1, BLEU, and graph edit distance to measure how accurately models recover schema elements such as entities, relationships, attributes, and extended ER constructs.","whyItMatters":"ER diagrams are central to database design but their image-based nature impedes automated processing. Existing VLM benchmarks do not focus on structured schema extraction from ERDs, and no public benchmark provides a standardized protocol for this task. ERUnderstand fills that gap with a reusable dataset and evaluation toolkit, enabling reproducible comparison of VLM performance on conceptual database schemas.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"90f18217e551a6266cad1ded9eeda84755e1af9dbf872ef664ec3fb6683db4e5"},"motivation":"Entity-Relationship Diagrams (ERDs) are central to conceptual database design, yet they are typically available only as rendered images rather than machine-readable schemas, limiting AI-assisted database engineering.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24707","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_dailybench_da2d9ccf","familyId":"bmf_8f2e299ee284","name":"DailyBench","oneLine":"DailyBench is a unified benchmark for evaluating AI-generated image detectors on modern full-image synthesis and object-level manipulation. It comprises FakeBench, with images from recent open-source and commercial generative models, and ManipulationBench, with subtle edits to real images. Detectors are scored by balanced accuracy.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.24016","pdf":"https://arxiv.org/pdf/2607.24016","project":"https://dailybench.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.24016"},"evidence":{"snippet":"To bridge this gap, we introduce DailyBench, a high-quality unified benchmark for evaluating whether AI-generated image detectors can generalize across both modern full-image synthesis and object-level manipulation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24016"},"ranking":{"30d":{"score":47,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"DailyBench is a unified benchmark for evaluating AI-generated image detectors on modern full-image synthesis and object-level manipulation. It comprises FakeBench, with images from recent open-source and commercial generative models, and ManipulationBench, with subtle edits to real images. Detectors are scored by balanced accuracy.","whyItMatters":"Existing detection benchmarks lag behind current generative models, causing a gap between evaluation and real-world scenarios. DailyBench provides a realistic testbed to assess generalization to modern synthesis and manipulation, offering practical value for developing robust detectors.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"829a2752de5a20de0a54a1781dc1e85fcb0bdf831538d19dd4c1a0eebd9e4c94"},"motivation":"Recent advances in generative models have shifted AI-generated image detection from identifying easily distinguishable, fully synthetic images to identifying highly realistic content generated by both modern generation and manipulation pipelines.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24016","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DailyBench Project","organizationType":"academic-lab","sourceUrl":"https://dailybench.github.io/","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_armnetbench_46ca6487","familyId":"bmf_4ef71bcd8184","name":"ArmnetBench","oneLine":"A benchmark for robot manipulation policies evaluated on a fleet of low-cost SO-101 cells. It compares 7 policies across 12 tasks in single-arm and bimanual configurations, with 2,518 policy rollouts and 600 reference demonstrations, all labeled successful, suboptimal, or failure. Data is released in LeRobot and RoboMeter formats.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.24481","pdf":"https://arxiv.org/pdf/2607.24481","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.24481"},"evidence":{"snippet":"We introduce ArmnetBench v0.1, a benchmark run on a fleet of low-cost SO-101 cells under light on-site supervision.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24481"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark for robot manipulation policies evaluated on a fleet of low-cost SO-101 cells. It compares 7 policies across 12 tasks in single-arm and bimanual configurations, with 2,518 policy rollouts and 600 reference demonstrations, all labeled successful, suboptimal, or failure. Data is released in LeRobot and RoboMeter formats.","whyItMatters":"Real-world evaluation of manipulation policies is costly and difficult to standardize. This benchmark provides a shared, public protocol with quality-labeled data, enabling comparable assessment of policies and supporting research on learning from mixed-quality demonstrations.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a5f574ec5d10c999c1cc8bac092f80a050edc49005a54385e59a5a2e52da5215"},"motivation":"Real-world evaluation is a bottleneck in developing generalist robot manipulation policies.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24481","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ArmnetBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.24481","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_aps-bench_6b2027af","familyId":"bmf_99de9ecb924b","name":"APS-Bench","oneLine":"APS-Bench is a 50-question QA dataset with auditable gold answers for evaluating retrieval-augmented generation over scientific facility operations knowledge.","area":"Language & Knowledge","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Research Infrastructure"],"capabilities":[],"topics":["physics.acc-ph"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.24663","pdf":"https://arxiv.org/pdf/2607.24663","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.24663"},"evidence":{"snippet":"We construct APS-Bench, a 50-question, question-answering (QA) dataset with auditable gold answers.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24663"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"APS-Bench is a 50-question QA dataset with auditable gold answers for evaluating retrieval-augmented generation over scientific facility operations knowledge.","whyItMatters":"It addresses the need for evaluating RAG systems on niche institutional knowledge, but its small scale and facility-specific scope limit broader utility.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3ffc8411e7e3445d17ae4e9458ab9a3c8a7f49ffaf4ba3fda450efe677ef3d2d"},"motivation":"Scientific user facilities accumulate decades of operational knowledge that no single search index covers: electronic logbooks, technical documents, internal wikis, operations chat messages, maintenance records, and live control-system data.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24663","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_agent-retrieval-bench_1105bc27","familyId":"bmf_0f8de99c2972","name":"Agent Retrieval Bench","oneLine":"File-level retrieval benchmark for coding agents, covering four positive tasks (code2test, comment2context, trace2code, edit2ripple) and a selective-retrieval subset with natural no-gold and counterfactual controls across 25 repositories, 427 samples, with frozen base-commit corpora.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Information retrieval"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.24882","pdf":"https://arxiv.org/pdf/2607.24882","project":null,"code":"https://github.com/eyuansu62/agent-retrieval-bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.24882"},"evidence":{"snippet":"We introduce Agent Retrieval Bench, a file-level benchmark for this upstream retrieval problem.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":6,"hfDailySubmittedAt":"2026-07-29T00:00:00.000Z","githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24882"},"ranking":{"30d":{"score":66,"rank":36,"coverage":0.85,"confidence":"High"},"90d":{"score":56,"rank":162,"coverage":0.7,"confidence":"Medium"}},"description":"File-level retrieval benchmark for coding agents, covering four positive tasks (code2test, comment2context, trace2code, edit2ripple) and a selective-retrieval subset with natural no-gold and counterfactual controls across 25 repositories, 427 samples, with frozen base-commit corpora.","whyItMatters":"Provides a dedicated evaluation for the context-acquisition stage of coding agents, distinguishing retrieval quality from patch generation and offering a reusable protocol for comparing retrieval and selective abstention methods.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4729577774258c87bd81952797c80bdace3b32993b4c2e0c7e1075a47f3acc68"},"motivation":"Modern coding agents are usually evaluated by whether they eventually produce a correct patch, but patch generation depends on an earlier context-acquisition stage: finding the repository files needed for the task.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24882","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Agent Retrieval Bench team","organizationType":"academic-lab","sourceUrl":"https://github.com/eyuansu62/agent-retrieval-bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_xih-bench_b0306814","familyId":"bmf_a796a6dc7c9f","name":"XIH-Bench","oneLine":"Evaluates instruction hierarchy compliance in multilingual LLMs using same-language and cross-language conflicts across six languages, four domains (rule-following, safety, task-execution, persona), and three hierarchy types (system-user, system-tool, user-tool).","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-26","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.23545","pdf":"https://arxiv.org/pdf/2607.23545","project":null,"code":"https://github.com/g1moon/Language-Shapes-IH","data":null,"hfPaper":"https://huggingface.co/papers/2607.23545"},"evidence":{"snippet":"We introduce XIH-Bench, a benchmark for multilingual IH evaluation with both same-language and cross-language conflicts across six languages, four domains, and three IH settings.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23545"},"ranking":{"30d":{"score":47,"rank":84,"coverage":0.85,"confidence":"High"},"90d":{"score":39,"rank":300,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates instruction hierarchy compliance in multilingual LLMs using same-language and cross-language conflicts across six languages, four domains (rule-following, safety, task-execution, persona), and three hierarchy types (system-user, system-tool, user-tool).","whyItMatters":"Existing instruction hierarchy benchmarks are largely English-centric, leaving a gap in assessing multilingual safety and reliability. This benchmark provides a reusable protocol to measure how language choice affects model compliance, supporting safer deployment in multilingual contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a550351d2dc3302ee2fca157c9db57278a19262f3fd0cf3b99c44e2e41ffcd8c"},"motivation":"Instruction hierarchy (IH) requires models to prioritize instructions by source, ensuring that higher-priority instructions override lower-priority ones.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"EMNLP 2026 (Main)","evidence":"Accepted to EMNLP 2026 (Main). Code and data are available at https://github.com/g1moon/Language-Shapes-IH","evidenceUrl":"https://arxiv.org/abs/2607.23545","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"EMNLP 2026 (Main)","reviewStatus":"accepted","decisionRaw":"Accepted to EMNLP 2026 (Main). Code and data are available at https://github.com/g1moon/Language-Shapes-IH","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.23545","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to EMNLP 2026 (Main). Code and data are available at https://github.com/g1moon/Language-Shapes-IH","level":"author-claim"}]}],"publishers":[{"name":"g1moon","organizationType":"community","sourceUrl":"https://github.com/g1moon/Language-Shapes-IH","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_tla-bench_4cd98a83","familyId":"bmf_205d79c0e84d","name":"TLA+-Bench","oneLine":"TLA+-Bench evaluates natural-language to TLA+ specification generation by executing each specification in the TLA+ model checker across the full reachable state space. The dataset includes 403 model-checked gold and 897 parse-only silver specifications, with multiple descriptions and difficulty labels.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-26","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.23425","pdf":"https://arxiv.org/pdf/2607.23425","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.23425"},"evidence":{"snippet":"We present TLA$^{+}$-Bench, a dataset and benchmark that grades by execution.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23425"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TLA+-Bench evaluates natural-language to TLA+ specification generation by executing each specification in the TLA+ model checker across the full reachable state space. The dataset includes 403 model-checked gold and 897 parse-only silver specifications, with multiple descriptions and difficulty labels.","whyItMatters":"Prior benchmarks for formal specification generation grade by reference resemblance or parseability, not correctness. TLA+-Bench provides an execution-grounded oracle that measures whether generated specifications satisfy the required properties, offering a more reliable evaluation signal and revealing a range of correctness scores depending on grading choices.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0e83bddca92dc4f43b97bae05b00093ae43a88970802e95a4f9fbca826ea1ccc"},"motivation":"Large language models increasingly write TLA$^{+}$ formal specifications from natural-language descriptions, but progress is hard to measure: existing resources grade by resemblance to a reference or by whether the output parses, neither of which shows correctness.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.23425","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_sombench_39ae8a74","familyId":"bmf_06b31d962c2c","name":"SoMBench","oneLine":"SoMBench evaluates social intelligence in large language models across 3 primary dimensions, 17 secondary dimensions, and 71 task paradigms, with 284 shared scenarios and 3,481 expert-verified instances.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.23740","pdf":"https://arxiv.org/pdf/2607.23740","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.23740"},"evidence":{"snippet":"For measurement, we introduce SoMBench, a psychology-grounded benchmark spanning 3 primary dimensions, 17 secondary dimensions, and 71 task paradigms.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23740"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SoMBench evaluates social intelligence in large language models across 3 primary dimensions, 17 secondary dimensions, and 71 task paradigms, with 284 shared scenarios and 3,481 expert-verified instances.","whyItMatters":"SoMBench targets the gap in evaluating LLMs' social intelligence, providing a structured benchmark to measure capabilities that are increasingly important for long-term deployment in human environments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f4cb66738e796a59c9a6315c9f98b2ba928dc9b04da86d6947be750a2ca34778"},"motivation":"As large language models move from isolated task solving toward long-term service in human environments, they require social intelligence: the ability to infer mental states, track social relations, reason over norms, and adapt behavior under context.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.23740","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_obsdrivebench_13a5d40b","familyId":"bmf_75ddbe4da96d","name":"ObsDriveBench","oneLine":"ObsDriveBench evaluates multimodal understanding in autonomous driving under adverse weather, covering observability awareness, spatial reliability, and risk-aware decision-making with multiple-choice tasks over camera, LiDAR, and radar inputs.","area":"Multimodal","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Automotive"],"capabilities":[],"topics":["Multimodal"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-07-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.23537","pdf":"https://arxiv.org/pdf/2607.23537","project":null,"code":"https://github.com/russellyq/ObsDriveBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.23537"},"evidence":{"snippet":"To study this, we introduce \\textbf{ObsDriveBench}, a real-world multi-modal benchmark for adverse-weather autonomous driving.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23537"},"ranking":{"30d":{"score":56,"rank":55,"coverage":0.55,"confidence":"Low"},"90d":{"score":44,"rank":245,"coverage":0.55,"confidence":"Low"}},"description":"ObsDriveBench evaluates multimodal understanding in autonomous driving under adverse weather, covering observability awareness, spatial reliability, and risk-aware decision-making with multiple-choice tasks over camera, LiDAR, and radar inputs.","whyItMatters":"Existing benchmarks rarely assess vision-language models under real-world adverse conditions with multimodal inputs. ObsDriveBench targets this gap by providing a fine-grained diagnosis of model behavior when observations are unreliable, supporting safer autonomous driving systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1b189325d747f14dcd6725eb7c971dae0b02064d2bff6eeecd592514a1f01262"},"motivation":"Autonomous driving under adverse weather remains a critical challenge, yet existing vision-language benchmarks mainly evaluate under standard conditions, synthetic corruptions, or single modality.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.23537","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_mulrobbench_1e33deff","familyId":"bmf_65e0c7a5af31","name":"MulRobBench","oneLine":"An offline protocol-conditioned benchmark for Vision-Language-Action UAV agents, evaluating operational context understanding, multimodal evidence arbitration, degradation-aware reasoning, and risk-aware action planning across 3,024 samples with semantic scoring and structural diagnostics.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.23870","pdf":"https://arxiv.org/pdf/2607.23870","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.23870"},"evidence":{"snippet":"We introduce MulRobBench, an offline, protocol-conditioned benchmark for Vision-Language-Action (VLA) UAV agents in smart-city environments.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23870"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"An offline protocol-conditioned benchmark for Vision-Language-Action UAV agents, evaluating operational context understanding, multimodal evidence arbitration, degradation-aware reasoning, and risk-aware action planning across 3,024 samples with semantic scoring and structural diagnostics.","whyItMatters":"Most UAV benchmarks focus on perception or navigation, leaving a gap in assessing coupled physical evidence, protocol constraints, and action risk. MulRobBench's diagnostic dimensions could inform safety-critical UAV deployment decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7682d3bd8cd4ab6557a48eb7c2a9eca8ef94a2f94d0283ca218d970cdb977320"},"motivation":"Smart-city airspace is transforming Uncrewed Aerial Vehicles (UAVs) from passive sensing platforms into cyber-physical decision makers that must follow operational rules under degraded observations and ambiguous language.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.23870","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_indic-diarbench_c5a01338","familyId":"bmf_ad88e51e7898","name":"Indic DiarBench","oneLine":"Multilingual joint diarization and ASR benchmark for 22 Indian languages, with ~108 hours of human-corrected multi-speaker audio from meetings, far-field, and in-the-wild sources, including code-mixing and overlaps.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.23808","pdf":"https://arxiv.org/pdf/2607.23808","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.23808"},"evidence":{"snippet":"In this work, we introduce Indic DiarBench, a speaker diarization and ASR benchmark dataset spanning all 22 scheduled languages of India.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23808"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Multilingual joint diarization and ASR benchmark for 22 Indian languages, with ~108 hours of human-corrected multi-speaker audio from meetings, far-field, and in-the-wild sources, including code-mixing and overlaps.","whyItMatters":"Provides a standardized evaluation suite for speaker diarization and ASR on Indian languages, addressing a gap in multilingual speech technology assessment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"059b892921ca680858a0c22ceb66a8b681f85e60fb8bf68b2ce88a3f198b9401"},"motivation":"In this work, we introduce Indic DiarBench, a speaker diarization and ASR benchmark dataset spanning all 22 scheduled languages of India.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.23808","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Indic DiarBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.23808","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_earnings25_937fc49b","familyId":"bmf_c5e53bcda5ed","name":"Earnings25","oneLine":"Earnings25 is a finance-domain benchmark for evaluating automatic speech recognition (ASR) on English-language earnings calls. It includes testset-full (498 hours of S&P 500 earnings calls from Q4 2025) and testset-segmented (46 hours of 290 segments from 2025 U.S. earnings calls), with aligned transcripts and metadata like speaker roles and industry labels, enabling speaker- and industry-aware evaluation beyond WER.","area":"Speech & Audio","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-07-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.23813","pdf":"https://arxiv.org/pdf/2607.23813","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.23813"},"evidence":{"snippet":"We introduce Earnings25, a finance-domain benchmark for evaluating automatic speech recognition (ASR) on English-language earnings calls under realistic conditions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23813"},"ranking":{"30d":{"score":40,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Earnings25 is a finance-domain benchmark for evaluating automatic speech recognition (ASR) on English-language earnings calls. It includes testset-full (498 hours of S&P 500 earnings calls from Q4 2025) and testset-segmented (46 hours of 290 segments from 2025 U.S. earnings calls), with aligned transcripts and metadata like speaker roles and industry labels, enabling speaker- and industry-aware evaluation beyond WER.","whyItMatters":"Existing ASR benchmarks lack domain-specific, large-scale, and realistic finance data, and typically report only aggregate word error rate. Earnings25 provides a reproducible protocol and structured metadata to support more granular evaluation of ASR systems in the finance domain.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e37fc638ca9da3a66fa076f8aa9dabfc3e80673d78ab7a1750908710a039dc6f"},"motivation":"We introduce Earnings25, a finance-domain benchmark for evaluating automatic speech recognition (ASR) on English-language earnings calls under realistic conditions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.23813","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Earnings25 team","organizationType":"benchmark-organization","sourceUrl":"https://arxiv.org/abs/2607.23813","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_e-bench_c1793d08","familyId":"bmf_e19c9d617150","name":"E-Bench","oneLine":"E-Bench evaluates multi-step tool-use agents in synthetic state-changing tasks across three product domains: Honor of Kings, QQ Music, and Tencent Meeting. It requires agents to discover hidden information and compose multiple tool calls before changing state, with deterministic grading by database-state diffs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.23722","pdf":"https://arxiv.org/pdf/2607.23722","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.23722"},"evidence":{"snippet":"We introduce E-Bench, a fully synthetic benchmark with 323 state-changing tasks across three product domains: Honor of Kings, QQ Music, and Tencent Meeting.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23722"},"ranking":{"30d":{"score":47,"rank":null,"coverage":0.3,"confidence":"Low"},"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"E-Bench evaluates multi-step tool-use agents in synthetic state-changing tasks across three product domains: Honor of Kings, QQ Music, and Tencent Meeting. It requires agents to discover hidden information and compose multiple tool calls before changing state, with deterministic grading by database-state diffs.","whyItMatters":"E-Bench addresses the gap in evaluating complex tool-use agents that interact with stateful environments over multiple steps, providing a scalable and controllable alternative to existing benchmarks that often focus on isolated API calls or short trajectories.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b14b055828bb69da9e040feab6fc201b2d5d920996f91ee438b70cc1a5dd7e2b"},"motivation":"Large Language Models (LLMs) are increasingly deployed as agents that interact with stateful environments over multiple steps: gathering hidden information, composing tool calls, and committing state changes.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.23722","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_sqbench_9c1d0337","familyId":"bmf_ba6726f3c0ac","name":"SQBench","oneLine":"SQBench v1.0 evaluates language-model agents on 220 production-oriented tasks organized into L1 atomic capabilities, L2 composite skills, and L3 business scenarios. Tasks require processing input assets, using tools, and producing a specified deliverable. Scoring computes Completion, Risk Penalty, and Performance from a 10D Risk Matrix; Strict Pass requires Completion=1 and Risk Penalty=0.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.23123","pdf":"https://arxiv.org/pdf/2607.23123","project":null,"code":"https://github.com/shaqiu-ai/SQBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.23123"},"evidence":{"snippet":"We introduce SQBench, a benchmark for evaluating production-oriented task delivery by language-model agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23123"},"ranking":{"30d":{"score":34,"rank":138,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":361,"coverage":0.55,"confidence":"Low"}},"description":"SQBench v1.0 evaluates language-model agents on 220 production-oriented tasks organized into L1 atomic capabilities, L2 composite skills, and L3 business scenarios. Tasks require processing input assets, using tools, and producing a specified deliverable. Scoring computes Completion, Risk Penalty, and Performance from a 10D Risk Matrix; Strict Pass requires Completion=1 and Risk Penalty=0.","whyItMatters":"The benchmark targets delivery under domain constraints, a shared weakness in current models. Its scoring separates functional completion from risk, which could support decisions about agent deployment in production workflows. However, without public access to tasks and full results, its practical value is limited for external comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0a99db05505fe6f210c3dcaf8248dbaff264e6b7172899589d277490580ac512"},"motivation":"Existing evaluations of large language models cover knowledge, reasoning, coding, and tool use, but they rarely treat a verifiable deliverable produced within a constrained workflow as the unit of evaluation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.23123","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_parasgb_cd57a968","familyId":"bmf_3039ff119f9f","name":"ParasGB","oneLine":"ParasGB evaluates graph neural network predictions of parasitic capacitance and resistance on circuit graphs from analog/mixed-signal designs, with node-level ground capacitance, edge-level resistance, and edge-level coupling capacitance tasks.","area":"Language & Knowledge","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Semiconductors","Software & Cloud","Manufacturing"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.23225","pdf":"https://arxiv.org/pdf/2607.23225","project":null,"code":"https://github.com/ShenShan123/ParasGB.git","data":null,"hfPaper":"https://huggingface.co/papers/2607.23225"},"evidence":{"snippet":"To address this gap, we introduce ParasGB, the first open-source benchmark suite for pre-layout parasitic parameter prediction on circuit graphs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23225"},"ranking":{"30d":{"score":34,"rank":139,"coverage":0.55,"confidence":"Low"},"90d":{"score":29,"rank":362,"coverage":0.55,"confidence":"Low"}},"description":"ParasGB evaluates graph neural network predictions of parasitic capacitance and resistance on circuit graphs from analog/mixed-signal designs, with node-level ground capacitance, edge-level resistance, and edge-level coupling capacitance tasks.","whyItMatters":"ParasGB addresses the lack of public high-fidelity RC benchmarks for early parasitic estimation, enabling reproducible evaluation and development of GNN-based models for parasitic-aware design.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1802dfa3f3a0074748f8f7b7fa90cd963ce4e69b83906bf0eb4f5e84a3351d3f"},"motivation":"As chip manufacturing processes advance to deep submicron nodes, parasitic interconnect effects increasingly dominate the performance of analog and mixed-signal (AMS) circuits and often lead to costly layout iterations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.23225","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ShenShan123","organizationType":"community","sourceUrl":"https://github.com/ShenShan123/ParasGB.git","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_esf-bench_d9226708","familyId":"bmf_db75a85566d3","name":"ESF-Bench","oneLine":"ESF-Bench evaluates slot filling in enterprise contexts, covering 810 multi-turn dialogues and 6,530 slots across 8 domains, with a taxonomy of 57 challenging scenarios.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.23326","pdf":"https://arxiv.org/pdf/2607.23326","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.23326"},"evidence":{"snippet":"In this work, we introduce ESF-Bench, a challenging Enterprise Slot Filling benchmark consisting of 810 multi-turn samples and 6530 slots over 8 unique domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23326"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ESF-Bench evaluates slot filling in enterprise contexts, covering 810 multi-turn dialogues and 6,530 slots across 8 domains, with a taxonomy of 57 challenging scenarios.","whyItMatters":"Addresses the gap in evaluating LLMs for slot filling under real-world enterprise constraints and unexpected user behaviors, providing a standard for model comparison in this practical task.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2eb3e1b64b89447e78bf8bb9e9077712ec25218bcb3701efe1ea74d834ea5c3a"},"motivation":"The rapid rise of large language models (LLMs) has driven transformative adoption across enterprises.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.23326","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_allocbench_5aa496ac","familyId":"bmf_0c2c1932f1a4","name":"AllocBench","oneLine":"A paired benchmark tests whether LLM agents exhibit conscious allocation behavior under a fixed budget in an abstract text-based formulation and a code-construction task.","area":"Language & Knowledge","applicationDomains":["Robotics & Autonomous Systems","Finance & Economics"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics","Financial Services"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.23332","pdf":"https://arxiv.org/pdf/2607.23332","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.23332"},"evidence":{"snippet":"We introduce a paired benchmark that tests whether LLM agents exhibit conscious allocation behavior under a fixed budget in two contexts: an abstract text-based formulation and a code-construction task.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.23332"},"ranking":{"30d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"},"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A paired benchmark tests whether LLM agents exhibit conscious allocation behavior under a fixed budget in an abstract text-based formulation and a code-construction task.","whyItMatters":"It identifies a capability boundary in online tool allocation for frontier models, showing that abstract optimal behavior does not transfer to script-writing.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1d4e0d6a6294cdb78c5d1ec7059a285cd4ca4394e3357f83f88e31a6f0a4abc7"},"motivation":"Creating a reusable tool is an investment: an agent pays a fixed cost now in exchange for the potential of future reuse.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.23332","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"cross-domain"},{"id":"bm_wuyueval_f4ea028f","familyId":"bmf_e2fd7bd79b60","name":"WuYuEval","oneLine":"A multi-level benchmark evaluates LLMs in solid waste management across foundational knowledge, domain reasoning, and expert decision-making using closed-ended and open-ended questions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Factuality"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07529","pdf":"https://arxiv.org/pdf/2608.07529","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07529"},"evidence":{"snippet":"We introduce WuYuEval, a multi-level benchmark for evaluating LLMs in SWM across foundational knowledge, domain reasoning, and expert decision-making.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07529"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A multi-level benchmark evaluates LLMs in solid waste management across foundational knowledge, domain reasoning, and expert decision-making using closed-ended and open-ended questions.","whyItMatters":"Fills a gap in evaluating professional decisions under engineering, environmental, and policy constraints, providing a resource for developing domain-oriented models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c9abf8074a5ea6e32af3d836249e207b330a7244c21e5a6743930774785ce4ae"},"motivation":"Large language models (LLMs) are increasingly used as technical assistants, but their competence in solid waste management (SWM) remains difficult to assess because existing benchmarks emphasize general knowledge rather than professional decisions under engineering, environmental, and policy constraints.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07529","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_sceneactbench_da42bbe6","familyId":"bmf_9aedebf94723","name":"SceneActBench","oneLine":"A benchmark for visually conditioned action on complete multi-object 3D scenes across five tasks under a unified agent-environment loop, scored against hidden geometric ground truth.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.22393","pdf":"https://arxiv.org/pdf/2607.22393","project":null,"code":"https://github.com/Feinaldo2/SceneActBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.22393"},"evidence":{"snippet":"We present SceneActBench, a benchmark for visually conditioned action across five 3D tasks under a unified agent-environment loop.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":7,"hfDailySubmittedAt":"2026-07-27T00:00:00.000Z","githubStars":11,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.22393"},"ranking":{"90d":{"score":62,"rank":120,"coverage":0.7,"confidence":"Medium"}},"description":"A benchmark for visually conditioned action on complete multi-object 3D scenes across five tasks under a unified agent-environment loop, scored against hidden geometric ground truth.","whyItMatters":"Addresses the under-evaluation of agent action on complete 3D scenes, providing a comparative basis for VLM agents acting on 3D environments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"71c1287911aba64f84386cf017f29eb6f0743eb3d03f06eeaed0bc8953728f45"},"motivation":"Vision-language model (VLM) agents increasingly use tools to act on 3D scenes rather than only describe them.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.22393","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_nl2shacl-bench_8b61bc16","familyId":"bmf_95864a020f6a","name":"NL2SHACL-Bench","oneLine":"A benchmark suite for translating natural language requirements into SHACL shapes, evaluating semantic equivalence beyond string comparison.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.07530","pdf":"https://arxiv.org/pdf/2608.07530","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.07530"},"evidence":{"snippet":"To tackle these challenges, we present NL2SHACL-Bench, a benchmark suite for natural language to SHACL translation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.07530"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark suite for translating natural language requirements into SHACL shapes, evaluating semantic equivalence beyond string comparison.","whyItMatters":"Provides a meaningful basis for measuring advances in NL2SHACL translation, which is critical for domain experts authoring SHACL constraints.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"18d8452ba30257ec36936d3bd06d6d58874e99afb39f4f4fd8aba91cc13a3431"},"motivation":"SHACL is a core technology for validating the conformance of RDF knowledge graphs (KGs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.07530","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_missionbench_cc95e3a2","familyId":"bmf_a4dfacf50588","name":"MissionBench","oneLine":"A benchmark for mission-level evaluation of MLLMs in aerial 3D environments, consisting of 120 missions across five simulated environments and four task families.","area":"Vision & 3D","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Geometric reasoning"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.22014","pdf":"https://arxiv.org/pdf/2607.22014","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.22014"},"evidence":{"snippet":"We introduce MissionBench, a benchmark for mission-level evaluation of MLLMs in aerial 3D environments.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.22014"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark for mission-level evaluation of MLLMs in aerial 3D environments, consisting of 120 missions across five simulated environments and four task families.","whyItMatters":"Highlights the difficulty of zero-shot long-horizon embodied tasks and the need for closed-loop evaluation, motivating scaling-driven improvements.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0f2dbb9535534373f1396007bf234da9f9e3d2e467a1c29ae7ffaa72d8c2bfdb"},"motivation":"Multimodal Large Language Models (MLLMs) are emerging as core reasoning modules for embodied agents, yet it remains unclear how well general-purpose models can solve long-horizon embodied tasks from a single high-level instruction.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.22014","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_ir275k_3c0945b4","familyId":"bmf_291981a5e03d","name":"IR275K","oneLine":"A benchmark for infrared multi-frame super-resolution containing 594 video sequences and 275,196 frames with fixed sequence-level splits and a reproducible x4 evaluation protocol.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.22380","pdf":"https://arxiv.org/pdf/2607.22380","project":null,"code":"https://github.com/InfraRecon7/IR275K","data":null,"hfPaper":"https://huggingface.co/papers/2607.22380"},"evidence":{"snippet":"We introduce IR275K, a curated benchmark containing 594 infrared video sequences and 275,196 frames.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":21,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.22380"},"ranking":{"90d":{"score":63,"rank":112,"coverage":0.55,"confidence":"Low"}},"description":"A benchmark for infrared multi-frame super-resolution containing 594 video sequences and 275,196 frames with fixed sequence-level splits and a reproducible x4 evaluation protocol.","whyItMatters":"Provides a standardized evaluation resource for accuracy-efficiency trade-offs in infrared MFSR, a domain with fragmented evaluation practices.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"efe421530bdc246cb740bd0f3720a070429d1f4d7280987507d8973b28098e34"},"motivation":"Efficient processing is becoming increasingly important in infrared remote sensing, where satellite constellations produce large volumes of observations under constrained detector resolution, power, and downlink bandwidth.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.22380","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_dynabench_70c2d7da","familyId":"bmf_124c02b8d60a","name":"DynaBench","oneLine":"A benchmark for robot manipulation in dynamic environments, including dynamic manipulation and bimanual coordination tasks.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Agents","Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.22119","pdf":"https://arxiv.org/pdf/2607.22119","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.22119"},"evidence":{"snippet":"To rigorously evaluate these capabilities, we introduce DynaBench, a novel benchmark for robot manipulation in dynamic environments.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.22119"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark for robot manipulation in dynamic environments, including dynamic manipulation and bimanual coordination tasks.","whyItMatters":"Evaluates sample-efficient multi-agent cooperation in dynamic settings, addressing causal limitations in multi-stream policies.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0f96e13d92c5264485c7e8464a18c491cda9ec15d4b6ae64e089f255718710cb"},"motivation":"Multi-stream robot manipulation policies achieve unparalleled sample efficiency and generalization by modeling actions relative to environmental reference frames.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.22119","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_dba-bench_29cc8ff8","familyId":"bmf_800372eba78a","name":"DBA-Bench","oneLine":"A production-fidelity benchmark for LLM-based database operations agents, using instrumented PostgreSQL environments with active workloads and outcome-first evaluation across 106 scenarios.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.DB"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.22165","pdf":"https://arxiv.org/pdf/2607.22165","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.22165"},"evidence":{"snippet":"We present DBA-Bench, a benchmark addressing these gaps through production fidelity, outcome-first evaluation, and controlled scenario reproducibility.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.22165"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A production-fidelity benchmark for LLM-based database operations agents, using instrumented PostgreSQL environments with active workloads and outcome-first evaluation across 106 scenarios.","whyItMatters":"Addresses gaps between evaluation and production database operations, offering a reproducible basis for comparing agent safety and efficacy.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3137dc01d4ef08b28fe1e02666d7a60477ec9d9c65946916cc9e7a6ea2f60dc7"},"motivation":"LLM-based database agents show promise, but differing task scopes, testbeds, and metrics hinder comparison.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.22165","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_vistr-bench_5dcce2bf","familyId":"bmf_744954eb90a9","name":"ViSTR-Bench","oneLine":"An evaluation suite of 1,340 video QA pairs across 15 subtasks assessing MLLM qualitative reasoning in dynamic scenes, covering motion perception, spatial relations, outcome prediction, and physical dynamics.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20868","pdf":"https://arxiv.org/pdf/2607.20868","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20868"},"evidence":{"snippet":"In this paper, we introduce the Visual Spatial-Temporal Reasoning Benchmark (ViSTR-Bench), a novel evaluation suite designed to systematically assess whether MLLMs can perform qualitative reasoning from continuous visual cues in dynamic scenes.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20868"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"An evaluation suite of 1,340 video QA pairs across 15 subtasks assessing MLLM qualitative reasoning in dynamic scenes, covering motion perception, spatial relations, outcome prediction, and physical dynamics.","whyItMatters":"Current MLLMs lag behind humans in intuitive spatial-temporal reasoning; this probe highlights those gaps but lacks a standalone reusable benchmark.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ae48f6b0717c5be8d9fc7c12b59a668785bbf50b112720db48735b9adf1332fe"},"motivation":"Multimodal Large Language Models (MLLMs) have achieved remarkable success across diverse expert-level tasks, but they still struggle with fundamental abilities that humans naturally develop through continuous observation of the real world, such as spatial perception and dynamic reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20868","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_tencent-workbuddy-bench_93d020ca","familyId":"bmf_c8a034f326c0","name":"Tencent WorkBuddy Bench","oneLine":"Multi-domain coding-agent benchmark with reverse-engineered tasks across Code, Web, Office, and Security. Each subset has its own scoring instrument; scores are not comparable across subsets.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20911","pdf":"https://arxiv.org/pdf/2607.20911","project":"https://workbuddybench.com/","code":"https://github.com/Tencent/workbuddy-bench","data":"https://huggingface.co/datasets/tencent/workbuddy-bench","hfPaper":"https://huggingface.co/papers/2607.20911"},"evidence":{"snippet":"We introduce Tencent WorkBuddy Bench, a multi-domain evaluation suite for coding agents; this report documents its construction methodology, scoring protocol, and a cross-model leaderboard.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":26,"hfDailySubmittedAt":"2026-07-24T00:00:00.000Z","githubStars":295,"githubScope":"benchmark_repo","hfDatasetDownloads":4352,"hfDatasetLikes":17},"source":{"type":"arxiv","id":"2607.20911"},"ranking":{"90d":{"score":95,"rank":2,"coverage":1.0,"confidence":"High","datasetDownloadRank":4,"datasetRankPopulation":51}},"description":"Multi-domain coding-agent benchmark with reverse-engineered tasks across Code, Web, Office, and Security. Each subset has its own scoring instrument; scores are not comparable across subsets.","whyItMatters":"Addresses contamination in coding-agent evaluation by constructing tasks not discoverable via web search, and provides a reproducible open-source framework for auditable third-party runs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e21fc569d4441078bda1a7ed61d71fc381a4d6b6c9af3cc933eb570a1f43a7f1"},"motivation":"We introduce Tencent WorkBuddy Bench, a multi-domain evaluation suite for coding agents; this report documents its construction methodology, scoring protocol, and a cross-model leaderboard.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20911","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_spatialgen-bench_3d213770","familyId":"bmf_54461512c988","name":"SpatialGen-Bench","oneLine":"ProVisE is a framework for evaluating image-generation models on spatial benchmarks by converting visual answers into structured predictions. SpatialGen-Bench is a diagnostic dataset of 470 samples across 14 spatial subtasks.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.21072","pdf":"https://arxiv.org/pdf/2607.21072","project":"https://zju-omniai.github.io/ProVisE/","code":"https://github.com/ZJU-OmniAI/ProVisE","data":null,"hfPaper":"https://huggingface.co/papers/2607.21072"},"evidence":{"snippet":"We further introduce SpatialGen-Bench, a curated diagnostic benchmark of 470 samples across 14 spatial subtasks, four capability levels, and diverse answer forms.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":37,"hfDailySubmittedAt":"2026-07-24T00:00:00.000Z","githubStars":23,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.21072"},"ranking":{"90d":{"score":70,"rank":72,"coverage":0.7,"confidence":"Medium"}},"description":"ProVisE is a framework for evaluating image-generation models on spatial benchmarks by converting visual answers into structured predictions. SpatialGen-Bench is a diagnostic dataset of 470 samples across 14 spatial subtasks.","whyItMatters":"Existing spatial benchmarks rely on text or coordinates, limiting image-generation models. ProVisE adapts visual answers to original metrics, enabling comparison. However, unclear if it is a standalone benchmark or a framework.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a7fd92873345ae54aa6f0b132092a7b892c6298b91082aa2bc60b6a21e25fd1f"},"motivation":"Spatial intelligence is essential for agents to move from static semantic understanding toward interacting with the physical world.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.21072","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_sciexplore_df25f831","familyId":"bmf_0a205bd9cb5c","name":"SciExplore","oneLine":"SciExplore evaluates scientific information-seeking and reasoning in LLMs and agents with four task types: scientific database navigation, ambiguous literature retrieval, missing reference completion, and cross-source structured knowledge synthesis, spanning 103 expert-curated tasks across more than ten scientific disciplines.","area":"Robotics & Embodied AI","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20926","pdf":"https://arxiv.org/pdf/2607.20926","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20926"},"evidence":{"snippet":"We introduce SciExplore, a benchmark designed to evaluate scientific information-seeking and reasoning capabilities of LLMs and agents.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20926"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SciExplore evaluates scientific information-seeking and reasoning in LLMs and agents with four task types: scientific database navigation, ambiguous literature retrieval, missing reference completion, and cross-source structured knowledge synthesis, spanning 103 expert-curated tasks across more than ten scientific disciplines.","whyItMatters":"Existing benchmarks emphasize general-domain retrieval or static QA, leaving a gap in assessing realistic scientific workflows. SciExplore's progressive task complexity provides a way to compare model capabilities in entity-level reasoning, document identification, evidence grounding, and domain-level synthesis, informing deployment in scientific research contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"57d2d7f78bd5ff16fcdce1215b7d6fa62bd3fddeb61418c5fc8340d0e45d5b2b"},"motivation":"Scientific research involves complex information-seeking and reasoning workflows across heterogeneous sources.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20926","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SciExplore Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.20926","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"general"},{"id":"bm_regretbench_a3a93900","familyId":"bmf_7ae5aa690b6f","name":"RegretBench","oneLine":"RegretBench evaluates clarification policies in multi-turn conversational LLMs, using hidden-intent tasks and a regret-based objective to measure value loss relative to a reference policy.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.21143","pdf":"https://arxiv.org/pdf/2607.21143","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.21143"},"evidence":{"snippet":"We introduce RegretBench, a multi-turn benchmark that evaluates clarification as policy behavior rather than isolated question quality.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.21143"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RegretBench evaluates clarification policies in multi-turn conversational LLMs, using hidden-intent tasks and a regret-based objective to measure value loss relative to a reference policy.","whyItMatters":"It addresses the evaluation gap in conversational AI by jointly measuring intent resolution, interaction cost, and stopping decisions, offering a more comprehensive assessment of clarification behavior.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"707046653d174b7998a50f86e7495e269011a32e152dc0eefb32c930442e8746"},"motivation":"Ambiguous user requests make clarification a sequential decision problem for conversational LLM assistants: they must decide whether to ask, what to ask, when to stop, and when to answer.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.21143","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_legalcitetrust_1777e861","familyId":"bmf_9504ffa72e56","name":"LegalCiteTrust","oneLine":"LegalCiteTrust evaluates citation trustworthiness in Chinese long-form legal research reports, assessing coverage, support, and citation-level existence, fidelity, and applicability.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20872","pdf":"https://arxiv.org/pdf/2607.20872","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20872"},"evidence":{"snippet":"We introduce LegalCiteTrust, a benchmark for evaluating citation trustworthiness in Chinese long-form legal research reports.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20872"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LegalCiteTrust evaluates citation trustworthiness in Chinese long-form legal research reports, assessing coverage, support, and citation-level existence, fidelity, and applicability.","whyItMatters":"Long-form legal research reports increasingly rely on LLMs, but citation trustworthiness is critical for legal accuracy. This benchmark addresses the gap by measuring whether citations are not only real but also accurate and applicable, providing a more nuanced evaluation than simple existence checks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dc710f6b8c90ca698f074b46bf34a378659c696fc4d2ad2f6f710e0cae2e1a56"},"motivation":"Long-form legal research reports increasingly rely on LLMs and agentic research systems, but their reliability depends not only on answering the task, but also on whether cited legal authorities are trustworthy.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20872","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_int-bench_bde8cb1b","familyId":"bmf_16d5485f629c","name":"Int-Bench","oneLine":"Int-Bench is a simulation-based evaluation for LLM intervention behavior in tutoring. It simulates a student solving problems across code debugging, mathematics, and brain teasers, with a teacher deciding whether, when, and how to intervene, measuring frequency, timing, and impact on task success and generalization.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.21306","pdf":"https://arxiv.org/pdf/2607.21306","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.21306"},"evidence":{"snippet":"Here, we introduce Int-Bench, a simulation-based benchmark for evaluating LLM interventions during learning.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.21306"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Int-Bench is a simulation-based evaluation for LLM intervention behavior in tutoring. It simulates a student solving problems across code debugging, mathematics, and brain teasers, with a teacher deciding whether, when, and how to intervene, measuring frequency, timing, and impact on task success and generalization.","whyItMatters":"The evaluation gap is that AI assistants may over-assist, hindering learning. This benchmark aims to quantify intervention timing and content, providing a method to compare models on supportive versus answer-giving behavior, which is critical for educational AI.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bda1c89cb908767226d3a31994d3965f6b3a84b8e8244752aeea4277fea5f40d"},"motivation":"Large language models (LLMs) are increasingly used as tutors and thought partners, helping users reason through problems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.21306","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_icae-bench_617decda","familyId":"bmf_0115589cfafb","name":"ICAE-Bench","oneLine":"ICAE-Bench evaluates coding agents on interactive project-building tasks. Agents receive a fuzzy product requirement and must clarify missing details via an automated user agent, then implement the project in a container. Scoring uses black-box tests and multi-dimensional diagnostics including functional correctness, semantic/API similarity, structural fidelity, design quality, and interaction quality.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.21217","pdf":"https://arxiv.org/pdf/2607.21217","project":null,"code":"https://github.com/ALEX-nlp/ICAE-EVAL","data":null,"hfPaper":"https://huggingface.co/papers/2607.21217"},"evidence":{"snippet":"In this paper, we introduce ICAE-Bench, a benchmark for evaluating coding agents under interactive project-building settings.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":7,"hfDailySubmittedAt":"2026-07-23T00:00:00.000Z","githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.21217"},"ranking":{"90d":{"score":56,"rank":160,"coverage":0.7,"confidence":"Medium"}},"description":"ICAE-Bench evaluates coding agents on interactive project-building tasks. Agents receive a fuzzy product requirement and must clarify missing details via an automated user agent, then implement the project in a container. Scoring uses black-box tests and multi-dimensional diagnostics including functional correctness, semantic/API similarity, structural fidelity, design quality, and interaction quality.","whyItMatters":"Existing coding benchmarks focus on static, fully specified tasks, leaving a gap for interactive, open-ended development. ICAE-Bench provides a reproducible protocol for measuring agent performance in transforming incomplete requirements into working software, which is increasingly relevant for real-world coding workflows.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1e44db164316f5d9ccafc513c5c1c94a95245d9fe5bf9c7fe2046f647e0c03fa"},"motivation":"The recent emergence of vibe-coding workflows is changing what coding agents are expected to do.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.21217","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ALEX-nlp","organizationType":"academic-lab","sourceUrl":"https://github.com/ALEX-nlp/ICAE-EVAL","role":"benchmark-publisher"}],"capabilityGroups":["Agents","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_hyperimagenet_06535482","familyId":"bmf_aa3a815479d6","name":"HyperImageNet","oneLine":"HyperImageNet is a dataset of 26,084 airborne hyperspectral image patches with 224 spectral bands and 138 fine-grained land-cover categories, providing raw imagery, pixel-level semantic labels, and object-level instance masks for semantic and instance segmentation, along with an open-environment evaluation protocol.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.21050","pdf":"https://arxiv.org/pdf/2607.21050","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.21050"},"evidence":{"snippet":"We present HyperImageNet, a large-scale benchmark for fine-grained hyperspectral land-cover understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.21050"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HyperImageNet is a dataset of 26,084 airborne hyperspectral image patches with 224 spectral bands and 138 fine-grained land-cover categories, providing raw imagery, pixel-level semantic labels, and object-level instance masks for semantic and instance segmentation, along with an open-environment evaluation protocol.","whyItMatters":"Existing hyperspectral benchmarks lack fine-grained categories and instance-level annotations; HyperImageNet enables evaluation of models on high-spatial-resolution imagery with strict spatial separation for open-environment generalization.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2f52e8aa95da4627200049213411900aa877d5beb7e35d47f46ebb75516cea7f"},"motivation":"We present HyperImageNet, a large-scale benchmark for fine-grained hyperspectral land-cover understanding.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.21050","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HyperImageNet Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.21050","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_guardianagentbench_89ff40f2","familyId":"bmf_6f6ba219145f","name":"GuardianAgentBench","oneLine":"GuardianAgentBench (GABench) is a benchmark of 580 agent scenarios across six domains, evaluated on three production frameworks (LangChain, LlamaIndex, Vectara), with multi-stage validation and five adversarial attack modes to assess tool-use correctness and safety.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.20982","pdf":"https://arxiv.org/pdf/2607.20982","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20982"},"evidence":{"snippet":"We present GuardianAgentBench (GABench), a benchmark of 580 scenarios across six domains evaluated on three production-ready frameworks: LangChain, LlamaIndex, and Vectara.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20982"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"GuardianAgentBench (GABench) is a benchmark of 580 agent scenarios across six domains, evaluated on three production frameworks (LangChain, LlamaIndex, Vectara), with multi-stage validation and five adversarial attack modes to assess tool-use correctness and safety.","whyItMatters":"LLM agents need structured evaluation of tool selection and failure modes; GABench provides a reusable framework with multiple attack modes and guardrail assessment, offering a fine-grained view of where agents fail.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"969e7de6bcde2872f638de42b7d9814e7c538c18998495eee238c33023901d95"},"motivation":"As large language model agents increasingly operate autonomously with access to tools and external environments, ensuring their safe and reliable behavior becomes critical.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20982","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GuardianAgentBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.20982","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_culturetalk-id_d7b2d28e","familyId":"bmf_e0a1bd69f078","name":"CultureTalk-ID","oneLine":"CultureTalk-ID is a dialogue-based benchmark for cultural commonsense in Indonesian and local languages, containing 4,496 culturally grounded dialogues across 11 languages and 13 topics, with three tasks: dialogue-based multiple-choice reasoning, culturally faithful translation, and language steering.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.21016","pdf":"https://arxiv.org/pdf/2607.21016","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.21016"},"evidence":{"snippet":"We introduce CultureTalk-ID, the first dialogue-based benchmark for cultural commonsense in Indonesian and its local languages, comprising 4,496 culturally grounded dialogues across 11 languages and 13 culturally salient topics, curated through a multi-stage human pipeline with native speakers to ensure authenticity.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.21016"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CultureTalk-ID is a dialogue-based benchmark for cultural commonsense in Indonesian and local languages, containing 4,496 culturally grounded dialogues across 11 languages and 13 topics, with three tasks: dialogue-based multiple-choice reasoning, culturally faithful translation, and language steering.","whyItMatters":"Existing cultural benchmarks use isolated prompts; CultureTalk-ID captures cultural nuances in dialogue context, enabling evaluation of models' cultural understanding, transfer, and generation in real conversational settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dd83bfa1dde99e387ddcae17a7fecc17a43d73f72554f26562f0cf44eea78da7"},"motivation":"Culture is lived through conversation, yet existing Indonesian cultural commonsense benchmarks evaluate LLMs on short and isolated prompts, stripping away the dialogic context in which cultural nuances actually surface.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.21016","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"CultureTalk-ID Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.21016","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_crag-mm-diagnostics_8195c595","familyId":"bmf_572d27c0d445","name":"CRAG-MM-Diagnostics","oneLine":"CRAG-MM-Diagnostics is a diagnostic benchmark for knowledge-intensive visual question answering (KI-VQA) with stage-wise annotations to isolate visual grounding, object identification, and knowledge retrieval/reasoning, including metadata like target ROIs and visual complexity scores.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Information retrieval","Factuality"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.21155","pdf":"https://arxiv.org/pdf/2607.21155","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.21155"},"evidence":{"snippet":"To analyze the full KI-VQA pipeline, we introduce CRAG-MM-Diagnostics, a diagnostic benchmark with stage-wise data annotations that isolate 1) language-based visual grounding, 2) object identification, and 3) knowledge retrieval and reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.21155"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CRAG-MM-Diagnostics is a diagnostic benchmark for knowledge-intensive visual question answering (KI-VQA) with stage-wise annotations to isolate visual grounding, object identification, and knowledge retrieval/reasoning, including metadata like target ROIs and visual complexity scores.","whyItMatters":"End-task accuracy alone obscures failure sources in KI-VQA; this benchmark enables stage-wise analysis to identify bottlenecks, guiding improvements in multimodal retrieval-augmented generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"992d635f361217d634d58c3a48155ba4b595996d3322d10dcf5c4d9e9cf7b0d3"},"motivation":"Knowledge-Intensive Visual Question Answering (KI-VQA) benchmarks evaluate Vision-Language Models (VLMs) as multimodal knowledge assistants by requiring external information beyond a provided image to answer questions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026","evidence":"Accepted to ECCV 2026","evidenceUrl":"https://arxiv.org/abs/2607.21155","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026","reviewStatus":"accepted","decisionRaw":"Accepted to ECCV 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.21155","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to ECCV 2026","level":"author-claim"}]}],"publishers":[{"name":"CRAG-MM-Diagnostics Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.21155","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception","Search & Retrieval"],"domainScope":"general"},{"id":"bm_copyright-bench_ed85deec","familyId":"bmf_b961b5acbe95","name":"Copyright-Bench","oneLine":"Copyright-Bench evaluates LLM agents' compliance with copyright law through realistic commercial tasks—website development, merchandise design, and pitch deck production—where agents choose between public-domain and copyrighted content, with prompt variations and time pressure.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.21799","pdf":"https://arxiv.org/pdf/2607.21799","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.21799"},"evidence":{"snippet":"To that end, we introduce Copyright-Bench, a benchmark designed to evaluate LLM agents' compliance with copyright law.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.21799"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Copyright-Bench evaluates LLM agents' compliance with copyright law through realistic commercial tasks—website development, merchandise design, and pitch deck production—where agents choose between public-domain and copyrighted content, with prompt variations and time pressure.","whyItMatters":"As agents perform commercial tasks, legal compliance is critical; this benchmark provides a structured way to assess whether agents select copyrighted materials appropriately, informing safety and regulatory considerations.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b72fd4bfbfb60c408f947409917b855deeae0c9bee1eafde3d3de1cb7ba7c860"},"motivation":"Large language model (LLM) agents increasingly perform commercial tasks that involve retrieving external content, such as images, and, where appropriate, reproducing that content.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.21799","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Copyright-Bench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.21799","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_convbench_5f241ad0","familyId":"bmf_0c7864d7753a","name":"ConVBench","oneLine":"ConVBench is a vision-centric reasoning benchmark where each image is paired with two logically equivalent questions across six categories (action/state, complex counting, spatial reasoning, causal/intent, commonsense, temporal perception), with metrics for logical consistency and robust accuracy.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.21722","pdf":"https://arxiv.org/pdf/2607.21722","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.21722"},"evidence":{"snippet":"We introduce ConVBench, a complex vision-centric reasoning benchmark in which each image is paired with two logically equivalent questions across six categories: action and state, complex counting, spatial reasoning, causal and intent understanding, commonsense reasoning, and temporal perception.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.21722"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ConVBench is a vision-centric reasoning benchmark where each image is paired with two logically equivalent questions across six categories (action/state, complex counting, spatial reasoning, causal/intent, commonsense, temporal perception), with metrics for logical consistency and robust accuracy.","whyItMatters":"Reliable visual reasoning requires not just correctness but consistency; this benchmark measures both, offering a more rigorous test for LVLMs and motivating consistency-aware training methods.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f9c422bc1dad18c37d210b81c72bd8481f28d5f63eadc4719cd77b3151f6f9d7"},"motivation":"While Large Vision-Language Models (LVLMs) exhibit strong perceptual capabilities, they remain vulnerable in visual reasoning tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.21722","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ConVBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.21722","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_axis_d184e18d","familyId":"bmf_d39c118729fe","name":"AXIS","oneLine":"AXIS is a community-driven data engine and benchmark for robot manipulation, providing browser-based teleoperation for data collection, automated task generation and validation, and a dataset of 207 tasks and 50K+ trajectories, with a systematic held-out protocol for policy evaluation.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.21588","pdf":"https://arxiv.org/pdf/2607.21588","project":"https://axisaiorg.github.io/AXIS-V1/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.21588"},"evidence":{"snippet":"We present AXIS, a growable community-driven data engine and benchmark for scalable robot learning, which enables browser-based teleoperation for large-scale demonstration collection, automatically generates and validates new manipulation tasks, and transforms community-collected demonstrations into training-ready data through automated success checking, quality filtering, trajectory smoothing, and visual and physics-based augmentation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.21588"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"AXIS is a community-driven data engine and benchmark for robot manipulation, providing browser-based teleoperation for data collection, automated task generation and validation, and a dataset of 207 tasks and 50K+ trajectories, with a systematic held-out protocol for policy evaluation.","whyItMatters":"Scaling robot learning requires diverse data and standardized evaluation; AXIS offers a growable pipeline and unified evaluation suite, enabling comparison of VLA policies and studying scaling behavior.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d0ee8d9fcfc697abd47c25111f1da6802acfd57eb6b45f1e34e1013bf926d10d"},"motivation":"Learning effective robot manipulation policies requires diverse, high-quality demonstrations, yet existing data pipelines are often difficult to scale because they rely on specialized hardware, centralized operators, or fixed task suites.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.21588","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AXIS Team","organizationType":"academic-lab","sourceUrl":"https://axisaiorg.github.io/AXIS-V1/","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_waveformqa_f89d742c","familyId":"bmf_7ce4ac0aa81a","name":"WaveformQA","oneLine":"WaveformQA is a QA benchmark for LLM temporal reasoning over digital waveforms, comprising 360 questions with programmatically generated ground truths across eight categories, including multi-signal correlation and event ordering, with waveforms generated from open-source designs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20638","pdf":"https://arxiv.org/pdf/2607.20638","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20638"},"evidence":{"snippet":"This paper presents WaveformQA, an open-source question-answering benchmark for evaluating LLM temporal reasoning over digital waveforms.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20638"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"WaveformQA is a QA benchmark for LLM temporal reasoning over digital waveforms, comprising 360 questions with programmatically generated ground truths across eight categories, including multi-signal correlation and event ordering, with waveforms generated from open-source designs.","whyItMatters":"Temporal reasoning over waveforms is critical for hardware verification; existing benchmarks focus on HDL generation, leaving this capability untested. WaveformQA provides a reproducible way to evaluate LLMs on this task.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"83705100d08ccc14f829553cc7676c0203b024ca7d1b0a7a3f1f30470eaec649"},"motivation":"Large Language Models (LLMs) have demonstrated strong capabilities in code generation and reasoning, yet their ability to perform temporal reasoning over digital waveform data remains largely unexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20638","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"WaveformQA Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.20638","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_selectbench_07886b3c","familyId":"bmf_4efc6a551cf2","name":"SelectBench","oneLine":"SelectBench evaluates selective evidence adoption in retrieval-augmented language models, focusing on rejecting deceptive content. The benchmark includes a 325-example test set and rule- or judge-based reward scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20090","pdf":"https://arxiv.org/pdf/2607.20090","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20090"},"evidence":{"snippet":"We introduce SelectBench, a controlled benchmark and training set for selective evidence adoption, and post-train Qwen3.5-4B directly with DAPO using either deterministic rule rewards or a frozen semantic judge.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20090"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SelectBench evaluates selective evidence adoption in retrieval-augmented language models, focusing on rejecting deceptive content. The benchmark includes a 325-example test set and rule- or judge-based reward scoring.","whyItMatters":"Retrieval-augmented models often face mixed contexts with misleading content. A standardized evaluation for selective evidence adoption helps measure safety and reliability in real-world deployments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d1030938d7ac42af52becfd0cf3cb6f9321d83525ae43d464c3715c1482d7197"},"motivation":"Retrieval-augmented large language models frequently face contexts that interleave useful evidence with misleading statements or instruction-like content.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20090","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_rs-rie-bench_f4bb954a","familyId":"bmf_71850eabb7fa","name":"RS-RIE-Bench","oneLine":"RS-RIE-Bench evaluates reasoning-guided remote sensing image editing across temporal, causal, and spatial reasoning tasks, with metrics for region plausibility, preservation, and quality consistency.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20197","pdf":"https://arxiv.org/pdf/2607.20197","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20197"},"evidence":{"snippet":"To fill this gap, we introduce RS-RIE-Bench, the first benchmark for reasoning-guided remote sensing image editing.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20197"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RS-RIE-Bench evaluates reasoning-guided remote sensing image editing across temporal, causal, and spatial reasoning tasks, with metrics for region plausibility, preservation, and quality consistency.","whyItMatters":"Current image editing benchmarks focus on natural images, leaving a gap for remote sensing domains that require geographic reasoning and sensor consistency. A dedicated benchmark supports progress in specialized editing models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b14d8d6d408ab61824df1d42d792769a04517dc92294389f72b5c235f2d03768"},"motivation":"Remote sensing image editing aims to modify remote sensing images according to natural language instructions while preserving geographic rules and sensor observation characteristics.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20197","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_openskillrisk_99823415","familyId":"bmf_2b6a63f1b968","name":"OpenSkillRisk","oneLine":"OpenSkillRisk evaluates LLM-based agents on their ability to recognize and avoid safety risks when using third-party skills, with 263 risky skills in seven threat categories and sandboxed execution.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.20121","pdf":"https://arxiv.org/pdf/2607.20121","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20121"},"evidence":{"snippet":"To support quantitative and qualitative evaluation, we construct OpenSkillRisk, a dedicated safety benchmark containing 263 risky skills collected from public skill marketplaces.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20121"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"OpenSkillRisk evaluates LLM-based agents on their ability to recognize and avoid safety risks when using third-party skills, with 263 risky skills in seven threat categories and sandboxed execution.","whyItMatters":"Third-party skills can introduce latent security vulnerabilities. A dedicated safety benchmark helps assess agent risk reasoning and execution control in open-world scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"791382c5c098f6c0ad65124b0b4e064910f17fc1d230a5537d62d8fe3f2f2a30"},"motivation":"LLM-based agents leverage third-party skills to extend their capabilities in open-world scenarios.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20121","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_offnadirloc_b7680d27","familyId":"bmf_eb7ecdfb3ed6","name":"OffNadirLoc","oneLine":"OffNadirLoc evaluates UAV-to-satellite geo-localization under large off-nadir views, with structure-aware contextual weighting and view-coherent learning for viewpoint-invariant features.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19951","pdf":"https://arxiv.org/pdf/2607.19951","project":"https://montalario.github.io/offnadirloc/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19951"},"evidence":{"snippet":"In this work, we introduce OffNadirLoc, a new benchmark for large off-nadir UAV-to-satellite geo-localization.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19951"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OffNadirLoc evaluates UAV-to-satellite geo-localization under large off-nadir views, with structure-aware contextual weighting and view-coherent learning for viewpoint-invariant features.","whyItMatters":"Existing benchmarks focus on near-nadir views, limiting real-world deployment. A dedicated benchmark improves evaluation of large off-nadir scenarios with structural understanding and multi-view consistency.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cc0c6ac07c4c898f09b02d71c914f429186c66cefe5f77931db49b72bee18589"},"motivation":"Cross-view geo-localization between UAV and satellite imagery remains a fundamental yet highly challenging task, especially under large off-nadir views where drastic perspective distortions, occlusions, and appearance gaps occur.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19951","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"OffNadirLoc project","organizationType":"academic-lab","sourceUrl":"https://montalario.github.io/offnadirloc/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_navverse_d8d5fb61","familyId":"bmf_d28dc1824ef0","name":"NavVerse","oneLine":"NavVerse evaluates indoor-to-outdoor embodied navigation in continuous robot simulation, with 10,000 episodes across object, vision-language, and place navigation tasks.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19695","pdf":"https://arxiv.org/pdf/2607.19695","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19695"},"evidence":{"snippet":"We introduce NavVerse, a physics-enabled benchmark for indoor-to-outdoor embodied navigation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19695"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"NavVerse evaluates indoor-to-outdoor embodied navigation in continuous robot simulation, with 10,000 episodes across object, vision-language, and place navigation tasks.","whyItMatters":"Existing benchmarks evaluate indoor and outdoor navigation separately, missing cross-context challenges. A dedicated benchmark supports progress in integrated navigation scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"025fdf01ff32bd5cb253b2e1d67639c246d4b6e0c8db5028d745afe9d5b75636"},"motivation":"Robots deployed in delivery, campus, and emergency-response settings often need to navigate from buildings to streets within a single continuous episode.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19695","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_mv-bench_0da16fba","familyId":"bmf_eebf6373b8cd","name":"MV-Bench","oneLine":"MV-Bench evaluates multimodal language models on coordinating multi-view interface construction using Tableau workbooks, converting specifications into executable web interfaces.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19910","pdf":"https://arxiv.org/pdf/2607.19910","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19910"},"evidence":{"snippet":"We introduce MV-Bench, a benchmark for evaluating MLLMs on coordinated multi-view interface construction.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19910"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MV-Bench evaluates multimodal language models on coordinating multi-view interface construction using Tableau workbooks, converting specifications into executable web interfaces.","whyItMatters":"Multimodal models increasingly generate code from visual designs, but existing evaluations focus on single-chart generation. A dedicated benchmark assesses coordination and data semantics in multi-view interfaces.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0179b686f733b11b58d53f09353812ab823283c892bce208254d2819f6c7f385"},"motivation":"Multimodal large language models (MLLMs) are increasingly expected to automate visualization development by generating code directly from visual designs.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19910","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_lorft_f3adff80","familyId":"bmf_96f828c73143","name":"LoRFT","oneLine":"LoRFT evaluates long-range vehicle trajectory reconstruction from fixed highway cameras, with 6,601 manually verified trajectories and map-aware evaluation metrics like ADE and FDE.","area":"Vision & 3D","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Automotive"],"capabilities":["Factuality"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19911","pdf":"https://arxiv.org/pdf/2607.19911","project":null,"code":"https://github.com/YvfanZhu/LoRFT","data":null,"hfPaper":"https://huggingface.co/papers/2607.19911"},"evidence":{"snippet":"We introduce LoRFT, to our knowledge the first open benchmark dedicated to long-range vehicle trajectory reconstruction from fixed highway cameras.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19911"},"ranking":{"90d":{"score":49,"rank":204,"coverage":0.55,"confidence":"Low"}},"description":"LoRFT evaluates long-range vehicle trajectory reconstruction from fixed highway cameras, with 6,601 manually verified trajectories and map-aware evaluation metrics like ADE and FDE.","whyItMatters":"Long-range trajectories are essential for traffic safety and autonomous driving. A dedicated benchmark supports progress in reconstructing distant trajectories despite perspective compression and scale decay.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"09e502e90de8860f86c9bb97f6c9adc17727033f88f5923db33b385db1e45f42"},"motivation":"Long-range vehicle trajectories provide important spatio-temporal evidence for traffic safety analysis, autonomous driving evaluation, and data-driven traffic management, yet continuously recovering them from fixed highway cameras remains difficult.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19911","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LoRFT project","organizationType":"academic-lab","sourceUrl":"https://github.com/YvfanZhu/LoRFT","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_kinebench_86d2d206","familyId":"bmf_3667c8221a21","name":"KineBench","oneLine":"KineBench evaluates embodied world models via kinematic grounding, using 20 manipulation tasks in ManiSkill3 and metrics like spectral arc length and manipulability index.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.RO"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19876","pdf":"https://arxiv.org/pdf/2607.19876","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19876"},"evidence":{"snippet":"To reduce this ambiguity, we present KineBench, an IDM-free closed-loop benchmark for EWMs, built upon an explicit kinematic grounding pipeline.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19876"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"KineBench evaluates embodied world models via kinematic grounding, using 20 manipulation tasks in ManiSkill3 and metrics like spectral arc length and manipulability index.","whyItMatters":"Evaluating physical consistency of embodied world models is challenging. A benchmark without IDMs reduces attribution ambiguity, supporting reliable assessment of generated videos.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b1ff49f020c23985b3d7c0775dcd927d284ad1109fd9a1d7dcdb2b95736463b3"},"motivation":"Evaluating the physical consistency of embodied world models(EWMs) is a critical open challenge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19876","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_kernelgenbench_691527e5","familyId":"bmf_12af6c0291ed","name":"KernelGenBench","oneLine":"KernelGenBench evaluates LLM- and agent-generated Triton kernels across 210 operators from three sources (ATen, vLLM, cuBLAS) and six hardware platforms, with automatic accuracy verification and two evaluation tracks (LLM Pass@K and iterative agent generation).","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.27231","pdf":"https://arxiv.org/pdf/2607.27231","project":null,"code":"https://github.com/flagos-ai/KernelGenBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.27231"},"evidence":{"snippet":"We present KernelGenBench, a unified benchmark for systematically evaluating LLM- and agent-generated Triton kernels across diverse operator sources and heterogeneous hardware platforms.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":14,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.27231"},"ranking":{"90d":{"score":61,"rank":127,"coverage":0.55,"confidence":"Low"}},"description":"KernelGenBench evaluates LLM- and agent-generated Triton kernels across 210 operators from three sources (ATen, vLLM, cuBLAS) and six hardware platforms, with automatic accuracy verification and two evaluation tracks (LLM Pass@K and iterative agent generation).","whyItMatters":"Kernel generation is a specialized task lacking standardized evaluation; this benchmark provides a multi-source, multi-chip protocol to compare methods across diverse operators and hardware, enabling cost and portability assessment for autonomous kernel development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"05d9b8c03fdb9a325e3dd3710de2b9132f4d6a92b07747954a68759928335d5c"},"motivation":"Large language models (LLMs) have significantly increased the demand for efficient accelerator kernels, but kernel development remains a highly specialized and labor-intensive task.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.27231","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FlagOS AI","organizationType":"company-research-lab","sourceUrl":"https://github.com/flagos-ai/KernelGenBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_hallutruthqa_4b25f1be","familyId":"bmf_c7b1ef2118b6","name":"HalluTruthQA","oneLine":"HalluTruthQA is a fine-grained benchmark for Arabic QA hallucination detection, localization, and explanation, containing 2,400 expert-curated examples across four knowledge domains with span-level and explanation annotations.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20219","pdf":"https://arxiv.org/pdf/2607.20219","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20219"},"evidence":{"snippet":"We introduce HalluTruthQA, a fine-grained benchmark for hallucination evaluation in Arabic question answering.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20219"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"HalluTruthQA is a fine-grained benchmark for Arabic QA hallucination detection, localization, and explanation, containing 2,400 expert-curated examples across four knowledge domains with span-level and explanation annotations.","whyItMatters":"Hallucination evaluation typically uses response-level labels; this benchmark provides granular annotations to assess localization and explanation capabilities, but lacks a public reuse path for broader comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a8702053efd02a3f00deeb38ee306d006f272b7b6762eb650084998484884ac2"},"motivation":"Large language models (LLMs) can generate fluent Arabic answers, yet factual errors remain difficult to detect, localize, explain, and verify.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20219","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_frontier-financial-judgement_bf94dbc0","familyId":"bmf_3ef91cbe5c1c","name":"Frontier Financial Judgement","oneLine":"Frontier Financial Judgement assesses agents' ability to identify valuation-relevant financial information from 656 synthetic and real news items, matching expert labels.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.CL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20645","pdf":"https://arxiv.org/pdf/2607.20645","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20645"},"evidence":{"snippet":"We introduce Frontier Financial Judgement, a challenging new benchmark developed in collaboration with professional equity analysts to assess agents' ability to replicate expert human judgements.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20645"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Frontier Financial Judgement assesses agents' ability to identify valuation-relevant financial information from 656 synthetic and real news items, matching expert labels.","whyItMatters":"News-flow filtering is critical for equity analysts; the benchmark measures agent accuracy, cost, and false positives, but lacks public artifacts for reuse.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"df990acbabc7c6d58d3678a9cd84cf446dbc62cf331ae56dde739e4d0d2837e2"},"motivation":"We introduce Frontier Financial Judgement, a challenging new benchmark developed in collaboration with professional equity analysts to assess agents' ability to replicate expert human judgements.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20645","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_drgbt-1k_b65fb238","familyId":"bmf_71bb9565fcb3","name":"DRGBT-1K","oneLine":"DRGBT-1K is a large-scale benchmark for dynamic RGBT tracking with 1,045 real-world sequences, 795K frame pairs, dense annotations, and a unified evaluation protocol across 20 trackers, plus an online leaderboard.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19772","pdf":"https://arxiv.org/pdf/2607.19772","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19772"},"evidence":{"snippet":"5) We develop an online evaluation platform for DRGBT-1K and provide a leaderboard that collects all methods evaluated on this benchmark.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19772"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DRGBT-1K is a large-scale benchmark for dynamic RGBT tracking with 1,045 real-world sequences, 795K frame pairs, dense annotations, and a unified evaluation protocol across 20 trackers, plus an online leaderboard.","whyItMatters":"Dynamic modality and platform variations are underrepresented in tracking benchmarks; this provides systematic evaluation for robustness under real-world transitions, supporting tracker comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d36fb4d1380e516196afe7d479cb5c559c8828ccc7ef02fe1326b96164528943"},"motivation":"Dynamic RGBT (DRGBT) tracking aims to continuously localize a target when the available sensing modalities and observation platforms vary over time.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19772","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_d2vbench_a4547106","familyId":"bmf_c0b889ee24cf","name":"D2VBench","oneLine":"D2VBench evaluates LLM value alignment using 10,000 daily dilemma scenarios covering 158 fine-grained value concepts, with a hybrid paradigm of multiple-choice and open-ended questions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19834","pdf":"https://arxiv.org/pdf/2607.19834","project":null,"code":"https://github.com/tjunlp-lab/D2VBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.19834"},"evidence":{"snippet":"To address these issues, we propose D2VBench, a value alignment benchmark comprising 10,000 instances of real daily dilemma scenarios constructed through a multi-stage collaboration between LLMs and humans, grounded in 158 manually annotated fine-grained value concepts.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19834"},"ranking":{"90d":{"score":29,"rank":363,"coverage":0.55,"confidence":"Low"}},"description":"D2VBench evaluates LLM value alignment using 10,000 daily dilemma scenarios covering 158 fine-grained value concepts, with a hybrid paradigm of multiple-choice and open-ended questions.","whyItMatters":"Value alignment benchmarks often lack scenario diversity; this provides a large, fine-grained dataset for assessing alignment across value dimensions in realistic settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fe3c347a785663e72dacd23502fa9078d9753666f45d48c71ad1d40d9c38e963"},"motivation":"With the wide application of large language models (LLMs) in real-world scenarios, the value implication of their outputs is crucial.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19834","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Tianjin University NLP Lab","organizationType":"academic-lab","sourceUrl":"https://github.com/tjunlp-lab/D2VBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_arbigraph_c1f2133e","familyId":"bmf_ef00fea6017a","name":"ArbiGraph","oneLine":"ArbiGraph is a benchmark generator for evaluating tool-assisted language agents' context management via scalable task graphs with exact automatic verification.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20764","pdf":"https://arxiv.org/pdf/2607.20764","project":null,"code":"https://github.com/pavelgolikov/ArbiGraph.git","data":null,"hfPaper":"https://huggingface.co/papers/2607.20764"},"evidence":{"snippet":"We introduce ARBIGRAPH, a benchmark generator for evaluating whether tool-assisted language agents can retain, update, compose, and discard task-relevant context across extended reasoning workflows.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20764"},"ranking":{"90d":{"score":39,"rank":301,"coverage":0.7,"confidence":"Medium"}},"description":"ArbiGraph is a benchmark generator for evaluating tool-assisted language agents' context management via scalable task graphs with exact automatic verification.","whyItMatters":"Context management is critical for long reasoning workflows; this generator allows controlled variation of task complexity, but the public path is incomplete.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5163da202c15cdd5d75077b82ad4f8400b600dbd40b88577ea86a9150fac2b6a"},"motivation":"We introduce ARBIGRAPH, a benchmark generator for evaluating whether tool-assisted language agents can retain, update, compose, and discard task-relevant context across extended reasoning workflows.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20764","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_xl-docbench_d4b9f759","familyId":"bmf_7ec798d59d5d","name":"XL-DocBench","oneLine":"XL-DocBench evaluates evidence-grounded long-document understanding with 1,519 human-verified questions from six professional domains, contexts up to 2,303 pages, multi-page evidence, and typed reasoning rules.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences","Finance & Economics"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech","Financial Services"],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00036","pdf":"https://arxiv.org/pdf/2608.00036","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00036"},"evidence":{"snippet":"We introduce XL-DocBench, a fully human-verified benchmark for extra-long document understanding, with 1,519 retained questions from six professional domains and contexts up to 2,303 pages.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00036"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"XL-DocBench evaluates evidence-grounded long-document understanding with 1,519 human-verified questions from six professional domains, contexts up to 2,303 pages, multi-page evidence, and typed reasoning rules.","whyItMatters":"Professional workflows require traceable answers from long documents; this benchmark fills a gap in multi-page and structured reasoning evaluation, enabling failure attribution.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"364074b787831af6ba8349ef106978e3cd713d305573340cd522c517978db1d5"},"motivation":"Real-world document tasks often ask professionals to answer questions from annual reports, regulations, clinical guidelines, and technical manuals that span hundreds or thousands of pages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00036","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"cross-domain"},{"id":"bm_vector-bench_6366f6e3","familyId":"bmf_628ccde1c0ca","name":"Vector-Bench","oneLine":"Vector-Bench evaluates instruction-based SVG code editing with 40 repair tasks, deterministic specification rewards, and metrics including repair progress and unintended change rate, across 34 model endpoints.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19056","pdf":"https://arxiv.org/pdf/2607.19056","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19056"},"evidence":{"snippet":"We introduce Vector-Bench, a compact, difficult benchmark of 40 SVG repair tasks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19056"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Vector-Bench evaluates instruction-based SVG code editing with 40 repair tasks, deterministic specification rewards, and metrics including repair progress and unintended change rate, across 34 model endpoints.","whyItMatters":"Vector editing fidelity is under-evaluated; this benchmark provides precise specification metrics, but public availability is unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4ea72572a5660a2b2fe8e58d6ae2518f9bbae1eae024169476b004d7d32104b2"},"motivation":"Instruction-based vector editing requires two capabilities: making a requested change and leaving everything else alone.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19056","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_scihazard_d93c4c2a","familyId":"bmf_f9cf90b25fd3","name":"SciHazard","oneLine":"SciHazard evaluates LLMs on scientific safety risks across 12 disciplines with 2,400 hazardous and 600 oversafety questions grounded in regulated entities. The DeHarm-Score decomposes harm into executability and net-new risk, providing a detailed scoring contract.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.18665","pdf":"https://arxiv.org/pdf/2607.18665","project":"https://anonymous.4open.science/r/DeharmScore-7B55","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.18665"},"evidence":{"snippet":"To address this, we introduce SciHazard, a real-world-grounded benchmark for scientific risks and a dataset agnostic evaluation framework for measuring harmfulness.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.18665"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SciHazard evaluates LLMs on scientific safety risks across 12 disciplines with 2,400 hazardous and 600 oversafety questions grounded in regulated entities. The DeHarm-Score decomposes harm into executability and net-new risk, providing a detailed scoring contract.","whyItMatters":"Existing safety benchmarks often use templated queries and LLM-as-a-Judge without domain grounding. SciHazard offers real-world grounded evaluation, and the DeHarm-Score improves agreement with expert annotations by 90% over baselines, enabling more reliable safety measurement for scientific LLMs and agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"aefcba623125c8927f039fa89c0054a27f726f3bc8f10a4ff70e94fb3210c20e"},"motivation":"Large language models (LLMs) increasingly support science, but they can also convert hazardous scientific knowledge into actionable misuse guidance.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.18665","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DeharmScore GitHub","organizationType":"community","sourceUrl":"https://anonymous.4open.science/r/DeharmScore-7B55","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_pathagentbench_70491a6b","familyId":"bmf_70918a81327d","name":"PathAgentBench","oneLine":"PathAgentBench evaluates vision-language models on whole-slide pathology images across four capabilities: image-to-text matching, text-to-image retrieval, diagnostic-region localization, and multi-scale reasoning. It includes 1,822 TCGA WSIs and 17,135 diagnostic paths.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning","Information retrieval"],"topics":["Multimodal","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19261","pdf":"https://arxiv.org/pdf/2607.19261","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19261"},"evidence":{"snippet":"We introduce PathAgentBench, a benchmark for evaluating evidence-seeking vision-language models (VLMs) across four complementary capabilities: image-to-text matching for evidence interpretation, text-to-image retrieval for evidence verification, diagnostic-region localization for evidence acquisition, and multi-scale reasoning for evidence integration.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19261"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PathAgentBench evaluates vision-language models on whole-slide pathology images across four capabilities: image-to-text matching, text-to-image retrieval, diagnostic-region localization, and multi-scale reasoning. It includes 1,822 TCGA WSIs and 17,135 diagnostic paths.","whyItMatters":"Most pathology benchmarks use pre-cropped patches, not whole-slide exploration. PathAgentBench provides a unified framework with annotated paths, revealing a significant gap in evidence acquisition and supporting progress in autonomous WSI diagnosis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"164a62c1666c683614d9776c3857d004743532095c87209c9cd9c0e1c75bad41"},"motivation":"Whole-slide image (WSI) diagnosis requires identifying diagnostically relevant regions, examining them across magnifications, and integrating multi-scale evidence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19261","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Search & Retrieval"],"domainScope":"specific"},{"id":"bm_openrtag_074a3702","familyId":"bmf_3b58e1fac77d","name":"OpenRTAG","oneLine":"OpenRTAG evaluates text-attributed graph learning under nine degradation scenarios (sparsity, noise, imbalance) across nine TAG datasets and three downstream tasks. It provides a standardized testbed for robustness comparison.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Robustness"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19108","pdf":"https://arxiv.org/pdf/2607.19108","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19108"},"evidence":{"snippet":"To address this gap, we present OpenRTAG, a robustness benchmark for text-attributed graph learning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19108"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OpenRTAG evaluates text-attributed graph learning under nine degradation scenarios (sparsity, noise, imbalance) across nine TAG datasets and three downstream tasks. It provides a standardized testbed for robustness comparison.","whyItMatters":"Real-world TAGs suffer from data quality issues, but evidence on robustness is fragmented. OpenRTAG unifies these scenarios, enabling systematic evaluation and comparison of mitigation strategies across model families.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7aa7deeedbb15365d7e46d9f71344777e76b0a4480b78b7c454c16dab3719b1b"},"motivation":"Text-attributed graphs (TAGs) are an important graph data form that combine relational structure with rich node text.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19108","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_moving-alphabet_5bd200ea","familyId":"bmf_9c1339291416","name":"Moving Alphabet","oneLine":"Moving Alphabet is a procedural testbed for controlled experiments on text-to-video training data, generating synthetic videos with ground-truth metadata to study data distribution and caption quality effects.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.18789","pdf":"https://arxiv.org/pdf/2607.18789","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.18789"},"evidence":{"snippet":"To enable controlled experiments, we introduce Moving Alphabet, a procedural testbed that renders letters with varying fonts, colors, sizes, and positions, moving in different directions and speeds against a black background.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":"2026-07-23T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.18789"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Moving Alphabet is a procedural testbed for controlled experiments on text-to-video training data, generating synthetic videos with ground-truth metadata to study data distribution and caption quality effects.","whyItMatters":"It provides insights into data curation for text-to-video models, highlighting the importance of diverse distributions and caption quality, but it is not a model comparison benchmark.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2354ee3f9f8ea9b6123774383feef61664bc9ab43e70622b142ddf0dbd5c59c8"},"motivation":"Text-to-video generation has advanced significantly over the past five years through scaling of model size, data, and compute.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.18789","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_missingbench-verified_db33d9c0","familyId":"bmf_d84428e12b3d","name":"MissingBench-Verified","oneLine":"MissingBench-Verified evaluates vision-language models on detecting missing object parts in images, covering ten leading models with consistent failure rates and probing mitigation strategies.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.18673","pdf":"https://arxiv.org/pdf/2607.18673","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.18673"},"evidence":{"snippet":"We present MissingBench-Verified, a benchmark designed to evaluate a specific and practically relevant scenario: when vision-language models fail to recognize that an essential component of an object has been removed.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.18673"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MissingBench-Verified evaluates vision-language models on detecting missing object parts in images, covering ten leading models with consistent failure rates and probing mitigation strategies.","whyItMatters":"It highlights a fundamental limitation of VLMs for inspection tasks, showing that current prompting and post-hoc corrections are insufficient, which informs the need for architectural changes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"50ee98a9487b2fa81dab8a16cd28b7ce977e62d1839a8ff1543b23e60e7ce3c5"},"motivation":"Vision Language Models (VLMs) are well known for hallucinating non-existent objects in images.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.18673","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mira-ev_98696803","familyId":"bmf_29781ac889f4","name":"MIRA-Ev","oneLine":"MIRA-Ev evaluates clinical NLP on evidence detection and relational reasoning in Spanish MIR exam cases, with three tasks: evidence sentence retrieval, argumentative component extraction, and relation classification.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19201","pdf":"https://arxiv.org/pdf/2607.19201","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19201"},"evidence":{"snippet":"We introduce MIRA-Ev, a clinical argument mining benchmark built on Spanish M\\'edico Interno Residente (MIR) licensing-exam cases, re-annotated by expert clinicians with span-level premises, claims, and directed support/attack relations, and released in parallel Spanish (native), English, and Basque versions, the first clinical argumentation resource in Basque.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19201"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MIRA-Ev evaluates clinical NLP on evidence detection and relational reasoning in Spanish MIR exam cases, with three tasks: evidence sentence retrieval, argumentative component extraction, and relation classification.","whyItMatters":"It addresses the lack of evidence-level evaluation in clinical NLP, providing a multilingual benchmark (Spanish, English, Basque) to assess not just final answers but the grounding of diagnoses.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c0a11c9dcd59b70e231f8a97cab34962be656fa22530b8fa557992f07c849f6d"},"motivation":"Clinical NLP evaluation remains dominated by multiple-choice question answering (MCQA), which scores only final-answer accuracy and cannot detect when a model reaches the correct diagnosis while grounding it in irrelevant, absent, or contradictory evidence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19201","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_meetingtom_9e241987","familyId":"bmf_69648c95f671","name":"MeetingToM","oneLine":"MeetingToM evaluates multimodal LLMs on theory-of-mind reasoning in multi-party meetings, including pseudo-consensus detection, across three levels: subject, dyad, and group. It provides a unified evaluation protocol.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19235","pdf":"https://arxiv.org/pdf/2607.19235","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19235"},"evidence":{"snippet":"We introduce MeetingToM, a benchmark for complex social behavior reasoning in naturalistic multi-party meetings.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19235"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MeetingToM evaluates multimodal LLMs on theory-of-mind reasoning in multi-party meetings, including pseudo-consensus detection, across three levels: subject, dyad, and group. It provides a unified evaluation protocol.","whyItMatters":"It covers latent social states and group dynamics often missing in existing ToM benchmarks, revealing limitations in integrating non-verbal cues and inferring hidden attitudes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"68b984db09633bd4ed37499e16247e584a24a1785d41080dc103e8dc04511726"},"motivation":"Theory of Mind (ToM), the ability to infer other's beliefs, intentions, and states of knowledge, is central to social interaction, yet remains challenging for current Multimodal Large Language Models (MLLMs), especially in multi-party meetings where cues are distributed across speech and behavior.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19235","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_medddc-eval_84cc7de6","familyId":"bmf_7b4401a381b7","name":"MedDDC-Eval","oneLine":"MedDDC-Eval evaluates multi-turn medical consultation agents by decoupling diagnosis from the diagnostic reader, using a frozen shared reader to score policies. It reports diagnostic support, coverage, and efficiency across Record and Dialogue splits.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-07-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.18999","pdf":"https://arxiv.org/pdf/2607.18999","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.18999"},"evidence":{"snippet":"We introduce MedDDC-Eval, a diagnosis-decoupled evaluation testbed over held-out cases derived from medical records and online consultations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.18999"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MedDDC-Eval evaluates multi-turn medical consultation agents by decoupling diagnosis from the diagnostic reader, using a frozen shared reader to score policies. It reports diagnostic support, coverage, and efficiency across Record and Dialogue splits.","whyItMatters":"Coupled evaluation confounds history elicitation with terminal diagnosis. This testbed enables fair comparison and evaluation-informed policy optimization, providing a more reliable measure of diagnostic support.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"52783678b7ebb043c336d2412c93c1f7854b17231ee97d9c30a30cd8ffcdd4d1"},"motivation":"Evaluating multi-turn medical consultation agents requires judging the diagnostic support provided by the histories they elicit through interaction.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.18999","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_biosecbench-surveillance_edf01c2c","familyId":"bmf_bbdaab9ec888","name":"BioSecBench-Surveillance","oneLine":"BioSecBench-Surveillance evaluates AI agents on pathogen genomic surveillance across 100 tasks spanning seven categories, including taxonomic classification and genetic-engineering detection. Agents receive raw sequencing data and surveillance context, and their structured answers are graded deterministically against ground truth.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19262","pdf":"https://arxiv.org/pdf/2607.19262","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19262"},"evidence":{"snippet":"We present BioSecBench-Surveillance, a verifiable benchmark of 100 evaluations testing whether AI agents can infer the right analysis pipeline from raw sequencing data and surveillance context.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19262"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"BioSecBench-Surveillance evaluates AI agents on pathogen genomic surveillance across 100 tasks spanning seven categories, including taxonomic classification and genetic-engineering detection. Agents receive raw sequencing data and surveillance context, and their structured answers are graded deterministically against ground truth.","whyItMatters":"The benchmark addresses the lack of verifiable evaluation for AI agents in genomic surveillance, where analysis bottlenecks are emerging as data generation scales. It provides a standardized measure of agent reliability in critical public health applications, with practical value in assessing whether agents can be trusted for real-world outbreak response.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fe37268113d08f0cb80c2eafa3c68e1fa16d460b6d012be238bd6aa9d579811b"},"motivation":"As pathogen genomic surveillance scales, the bottleneck is shifting from data generation to analysis.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19262","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_wuyu-envle-bench_a002dca0","familyId":"bmf_cb77b53e6c10","name":"WuYu-EnvLE-Bench","oneLine":"WuYu-EnvLE-Bench evaluates LLMs on environmental law enforcement with 2,521 instances across 14 tasks and 12 pollution-medium subdomains, covering pre-, in-, and post-enforcement workflows. It uses Absolute Environmental Enforcement Score (AES) and Intelligent Enforcement Index (IEI) for evaluation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.17745","pdf":"https://arxiv.org/pdf/2607.17745","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.17745"},"evidence":{"snippet":"We introduce WuYu-EnvLE-Bench, a benchmark built from real enforcement cases, regulatory standards, and expert review.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.17745"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"WuYu-EnvLE-Bench evaluates LLMs on environmental law enforcement with 2,521 instances across 14 tasks and 12 pollution-medium subdomains, covering pre-, in-, and post-enforcement workflows. It uses Absolute Environmental Enforcement Score (AES) and Intelligent Enforcement Index (IEI) for evaluation.","whyItMatters":"The benchmark fills a gap in evaluating LLMs for evidence-grounded, rule-aware reasoning in environmental enforcement. It provides practical assessment of model reliability in traceable decision-making, highlighting limitations in evidence-chain construction and procedural judgment, which is critical for legal and regulatory applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f20497a9c0e3d2562c50f6ec3232b598e3967c5a446c8b6d4f08be77c4b3c331"},"motivation":"Large language models (LLMs) are increasingly considered for environmental enforcement, but their ability to produce traceable enforcement decisions remains unclear.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.17745","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_worldcuparena_3034c352","familyId":"bmf_4b5c0cb0434c","name":"WorldCupArena","oneLine":"WorldCupArena evaluates language models and deep-research agents on football forecasting across multiple tasks including result, score, player events, statistics, and tournament outcomes. It uses a composite score and supports adding new schedules for future events.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.18084","pdf":"https://arxiv.org/pdf/2607.18084","project":null,"code":"https://github.com/wzk1015/WorldCupArena","data":null,"hfPaper":"https://huggingface.co/papers/2607.18084"},"evidence":{"snippet":"We present WorldCupArena, a dynamic benchmark for language models and deep-research agents.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":5,"hfDailySubmittedAt":"2026-07-21T00:00:00.000Z","githubStars":24,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.18084"},"ranking":{"90d":{"score":67,"rank":89,"coverage":0.7,"confidence":"Medium"}},"description":"WorldCupArena evaluates language models and deep-research agents on football forecasting across multiple tasks including result, score, player events, statistics, and tournament outcomes. It uses a composite score and supports adding new schedules for future events.","whyItMatters":"This benchmark provides a dynamic, real-world testbed for evaluating models on multi-source reasoning and structured prediction with ground truth on a fixed schedule. It offers practical value in comparing model performance against market and human baselines, revealing differences in detailed predictions beyond result accuracy.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3abb2ecf948b0e807d56a9402f9390a5f6a48869a5ecdabf830ae8e4c4f294ec"},"motivation":"Predicting a football match before kickoff requires more than knowing past results: a model must use changing information and make a clear prediction before the answer is available.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.18084","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_vehbench_bf410800","familyId":"bmf_06afd602ad70","name":"VEHBench","oneLine":"VEHBench is a diagnostic benchmark for LLM-assisted vibration energy harvester design, featuring 763 tasks across four design roles: specification triage, verifier-guided search, corrupted-state recovery, and policy-conditioned selection. It uses an analytical physical oracle for scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.18181","pdf":"https://arxiv.org/pdf/2607.18181","project":null,"code":null,"data":"https://huggingface.co/datasets/AnonymousVehbench/vehbench","hfPaper":"https://huggingface.co/papers/2607.18181"},"evidence":{"snippet":"We introduce VEHBench, an engineering-native diagnostic benchmark for LLM-assisted VEH design, featuring 763 literature-grounded tasks scored by an analytical physical oracle.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":57,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2607.18181"},"ranking":{"90d":{"score":42,"rank":253,"coverage":0.3,"confidence":"Low","datasetDownloadRank":39,"datasetRankPopulation":51}},"description":"VEHBench is a diagnostic benchmark for LLM-assisted vibration energy harvester design, featuring 763 tasks across four design roles: specification triage, verifier-guided search, corrupted-state recovery, and policy-conditioned selection. It uses an analytical physical oracle for scoring.","whyItMatters":"The benchmark addresses the need for stage-local evaluation of LLMs in coupled physical design workflows, revealing stage-dependent capabilities and response-control patterns. It provides practical value for selecting and routing models in engineering applications, highlighting that no single model dominates all stages.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"61fb0c8f13e3ab2a6f85fbcdd64789d095d4786248f6dc1a188b312d6f16662d"},"motivation":"Battery-free Internet of Things (IoT) requires iterative design of vibration energy harvesters (VEHs) under coupled physical constraints, while LLMs are emerging as interface layers for engineering workflows.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.18181","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_tcr-bench_7bc5c915","familyId":"bmf_567a72f9e197","name":"TCR-Bench","oneLine":"TCR-Bench is a diagnostic benchmark for table content-level answerability in RAG, focusing on sibling tables with similar schemas but content differences. It evaluates dense retrievers' ability to identify answerable tables.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-20","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.17742","pdf":"https://arxiv.org/pdf/2607.17742","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.17742"},"evidence":{"snippet":"To study this, we introduce TCR-Bench, a diagnostic benchmark for Table Content-level Answerability in RAG, built around sibling tables, i.e., tables with highly similar schemas but subtle content differences.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.17742"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TCR-Bench is a diagnostic benchmark for table content-level answerability in RAG, focusing on sibling tables with similar schemas but content differences. It evaluates dense retrievers' ability to identify answerable tables.","whyItMatters":"This benchmark probes the semantic-answerability gap in retrieval for table RAG, showing that semantic relevance does not guarantee answerability. It provides diagnostic insights into retriever limitations and the potential for answerability-aware reranking, but is primarily for research findings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8bfe57dfb5b78515896c5d95fdbf9fc687d5bd15576a08d6f520c55d592792f4"},"motivation":"Tables are a critical knowledge source in retrieval-augmented generation (RAG), but a retrieved table may lack sufficient evidence to answer a query, a property we call answerability.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.17742","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_relay-bench_1dc58a62","familyId":"bmf_4a246274f907","name":"Relay-Bench","oneLine":"Relay-Bench evaluates LLMs on multi-domain reasoning chains, presenting composite problems that combine subproblems from domains like visual reasoning, coding, math, information extraction, and data analysis. It measures overall accuracy on these complex, text-only tasks.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.18438","pdf":"https://arxiv.org/pdf/2607.18438","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.18438"},"evidence":{"snippet":"Introducing Relay-Bench, an unsaturated, holistic, text-only benchmark that measures LLMs' ability to complete an assortment of tasks from distinct domains in a single prompt.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.18438"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Relay-Bench evaluates LLMs on multi-domain reasoning chains, presenting composite problems that combine subproblems from domains like visual reasoning, coding, math, information extraction, and data analysis. It measures overall accuracy on these complex, text-only tasks.","whyItMatters":"The benchmark addresses the need for holistic evaluation of LLMs on combined reasoning across multiple domains, which is common in real-world tasks. It provides practical value in assessing models' ability to handle complex, multi-step problems, with current models scoring below 50%.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1f85cf7bede9e568ed47ccfe0ff17ebb4f859de5beedbc45cd78554f3c6b3dbe"},"motivation":"Introducing Relay-Bench, an unsaturated, holistic, text-only benchmark that measures LLMs' ability to complete an assortment of tasks from distinct domains in a single prompt.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.18438","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_proevent_18140068","familyId":"bmf_487414c0a9cc","name":"ProEvent","oneLine":"ProEvent is an event-centric benchmark for proactive agents, evaluating their ability to maintain a user's timetable from instant messaging chats. It assesses response timing, single-step correctness, and multi-step correctness using synthesized realistic chat scenarios.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.17701","pdf":"https://arxiv.org/pdf/2607.17701","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.17701"},"evidence":{"snippet":"To bridge these gaps, we introduce ProEvent, the first event-centric benchmark designed to assess an agent's ability to proactively maintain a user's timetable based on ongoing instant messaging chats.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.17701"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ProEvent is an event-centric benchmark for proactive agents, evaluating their ability to maintain a user's timetable from instant messaging chats. It assesses response timing, single-step correctness, and multi-step correctness using synthesized realistic chat scenarios.","whyItMatters":"The benchmark fills a gap in evaluating proactive agents for event-centric assistance, which is crucial for autonomous support. It provides practical value in measuring agents' ability to detect implicit events and reason from the user's perspective, revealing significant limitations in current models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1721fece9a4857fc0d2ef8370ed8fea2bbd1364392f26156e489f6dcae0781e4"},"motivation":"Proactive agents are expected to anticipate user needs and provide autonomous assistance by perceiving environmental context without explicit instructions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.17701","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_escucha_bdd66fa8","familyId":"bmf_07214793264e","name":"ESCUCHA","oneLine":"ESCUCHA is a Spanish speech understanding benchmark evaluating LALMs across heterogeneous acoustic conditions and reasoning abilities. It includes 1,000 human-curated audio-question pairs spanning perceptual and reasoning categories, with multiple accents and non-normative speech.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.17812","pdf":"https://arxiv.org/pdf/2607.17812","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.17812"},"evidence":{"snippet":"We introduce ESCUCHA, the first Spanish speech understanding benchmark designed to evaluate LALMs across heterogeneous acoustic conditions and reasoning abilities.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.17812"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ESCUCHA is a Spanish speech understanding benchmark evaluating LALMs across heterogeneous acoustic conditions and reasoning abilities. It includes 1,000 human-curated audio-question pairs spanning perceptual and reasoning categories, with multiple accents and non-normative speech.","whyItMatters":"The benchmark addresses the lack of robust evaluation for Spanish speech understanding under realistic conditions. It provides practical value in assessing model performance across diverse acoustic environments and reasoning tasks, highlighting gaps relative to human performance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"58de95993e993265778207d70af3e7fafb275cd603138bd1e7434fb757a5617e"},"motivation":"As large audio language models (LALMs) advance, robust evaluation frameworks have become essential.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.17812","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_choroplethmap-bench_3460d576","familyId":"bmf_92308d9f928f","name":"ChoroplethMap-Bench","oneLine":"ChoroplethMap-Bench evaluates spatial understanding of foundation models with 2,400 synthetic choropleth maps, corresponding GeoJSON data, and 12,000 questions across five cognitive dimensions (Identify, Spatial Recognition, Compare, Rank, Delineate). Models are assessed under Data Only, Map Only, and Data + Map conditions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.17999","pdf":"https://arxiv.org/pdf/2607.17999","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.17999"},"evidence":{"snippet":"We introduce ChoroplethMap-Bench, a controlled benchmark containing 2,400 synthetic choropleth maps, corresponding GeoJSON data, and 12,000 questions across five cognitive dimensions: Identify, Spatial Recognition, Compare, Rank, and Delineate.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.17999"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ChoroplethMap-Bench evaluates spatial understanding of foundation models with 2,400 synthetic choropleth maps, corresponding GeoJSON data, and 12,000 questions across five cognitive dimensions (Identify, Spatial Recognition, Compare, Rank, Delineate). Models are assessed under Data Only, Map Only, and Data + Map conditions.","whyItMatters":"The benchmark assesses whether cartographic representations add value over structured geodata for machine spatial reasoning, addressing a gap in evaluating map-based inputs. It supports decisions on when to incorporate visual map data in geospatial AI systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bcc47b172b981fc992a5061e574108d8fdd67c9465d2fc1200e6ecdf7d15c704"},"motivation":"Spatial understanding is crucial for foundation models (FMs), and maps have long helped humans organize and reason about geographic information.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.17999","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"The authors","organizationType":"community","sourceUrl":"https://arxiv.org/abs/2607.17999","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_autoworldmodel-bench_01d33f6e","familyId":"bmf_c9aa949f276e","name":"AutoWorldModel-Bench","oneLine":"AutoWorldModel-Bench evaluates coding agents on autonomously improving a base world model under fixed compute budget across eight game environments. Uses structured-state representation, with held-out test split and closed-loop iteration.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.11216","pdf":"https://arxiv.org/pdf/2608.11216","project":"https://electronicarts.github.io/AutoWorldModelBench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.11216"},"evidence":{"snippet":"We introduce AutoWorldModel-Bench, a closed-loop benchmark in which frontier coding agents autonomously improve a provided base world model under a fixed compute budget.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":13,"hfDailySubmittedAt":"2026-08-13T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.11216"},"ranking":{"90d":{"score":54,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"AutoWorldModel-Bench evaluates coding agents on autonomously improving a base world model under fixed compute budget across eight game environments. Uses structured-state representation, with held-out test split and closed-loop iteration.","whyItMatters":"Addresses the gap in agent benchmarks for open-ended research tasks, enabling comparison of agents on research-like workflows rather than engineering-to-spec tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5494aae2e8fbd6ee1e24e3e85e999963339b379d2b11d751c975eac82f277ae8"},"motivation":"World modeling is an unsettled field: architectures, training objectives, and state representations interact in complex ways, and no single recipe dominates across environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.11216","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Electronic Arts","organizationType":"company-research-lab","sourceUrl":"https://electronicarts.github.io/AutoWorldModelBench/","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_editclevr_a686a8c4","familyId":"bmf_5225fce0cba4","name":"EditCLEVR","oneLine":"EditCLEVR is a paired-scene intervention benchmark for object-centric representations, with before/after CLEVR renders and a known attribute change. Includes metrics like SGIA and Delta-SGIA for semantic faithfulness, with probe-free diagnostics.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-19","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.22705","pdf":"https://arxiv.org/pdf/2607.22705","project":null,"code":"https://github.com/torux-bughunter/EditCLEVR","data":null,"hfPaper":"https://huggingface.co/papers/2607.22705"},"evidence":{"snippet":"We introduce EditCLEVR, a paired-scene intervention benchmark in which each example contains a before/after pair of CLEVR-style renders with the same object indices and scene layout, and either exactly one known attribute change on one known object or a no-edit re-render for drift measurement.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.22705"},"ranking":{"90d":{"score":29,"rank":364,"coverage":0.55,"confidence":"Low"}},"description":"EditCLEVR is a paired-scene intervention benchmark for object-centric representations, with before/after CLEVR renders and a known attribute change. Includes metrics like SGIA and Delta-SGIA for semantic faithfulness, with probe-free diagnostics.","whyItMatters":"Provides a direct test of whether per-object representations behave correctly under controlled semantic edits, addressing a gap in evaluating compositional faithfulness beyond segmentation or single-image prediction.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"af332deae070394600265404b352ea94ee159983b428e0171ed4517e722d4687"},"motivation":"Object-centric learning aims to represent scenes as objects whose properties can be reused in new combinations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.22705","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"torux-bughunter","organizationType":"community","sourceUrl":"https://github.com/torux-bughunter/EditCLEVR","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_realdesed_918e47e6","familyId":"bmf_219e72d356ca","name":"RealDESED","oneLine":"RealDESED is a real-world domestic sound event detection benchmark with 5,710 recordings from 652 participants, 15 classes, and temporally precise annotations. Includes multi-annotator labeling and rich metadata.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["eess.AS"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.16736","pdf":"https://arxiv.org/pdf/2607.16736","project":null,"code":"https://github.com/fschmid56/RealDESED","data":"https://zenodo.org/records/20056072","hfPaper":"https://huggingface.co/papers/2607.16736"},"evidence":{"snippet":"This paper presents RealDESED, a real-world domestic sound event detection (SED) benchmark comprising 5,710 audio recordings collected by 652 participants in their homes.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.16736"},"ranking":{"90d":{"score":49,"rank":205,"coverage":0.55,"confidence":"Low"}},"description":"RealDESED is a real-world domestic sound event detection benchmark with 5,710 recordings from 652 participants, 15 classes, and temporally precise annotations. Includes multi-annotator labeling and rich metadata.","whyItMatters":"Provides a realistic alternative to synthetic or web-crawled SED datasets, supporting evaluation of systems under natural domestic conditions for deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9f39c76f10dcea7cf14d317e6dd5f15c4463ed6678466c2611b2fc418a251ac1"},"motivation":"This paper presents RealDESED, a real-world domestic sound event detection (SED) benchmark comprising 5,710 audio recordings collected by 652 participants in their homes.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.16736","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"The authors","organizationType":"community","sourceUrl":"https://zenodo.org/records/20056072","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_oct-bench_3914e5be","familyId":"bmf_b657b809f328","name":"OCT-Bench","oneLine":"OCT-Bench evaluates multimodal large language models on OCT image understanding with 10,076 multiple-choice questions from 4,137 images across seven public datasets, covering 20 tasks in Perception, Cognition, and Reasoning.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.16609","pdf":"https://arxiv.org/pdf/2607.16609","project":null,"code":"https://github.com/baochenfu/OCT-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.16609"},"evidence":{"snippet":"To address this limitation, we introduce OCT-Bench, a comprehensive benchmark dedicated to OCT image understanding.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":"2026-07-21T00:00:00.000Z","githubStars":145,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.16609"},"ranking":{"90d":{"score":78,"rank":26,"coverage":0.7,"confidence":"Medium"}},"description":"OCT-Bench evaluates multimodal large language models on OCT image understanding with 10,076 multiple-choice questions from 4,137 images across seven public datasets, covering 20 tasks in Perception, Cognition, and Reasoning.","whyItMatters":"Offers a comprehensive benchmark for OCT understanding beyond coarse classification, enabling capability bottleneck analysis for clinical deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"238c1136dffdc0b4d1084bc66bcb501e556b225bc0a4c86211000e3b45dc0ad4"},"motivation":"Optical coherence tomography (OCT) imaging is essential for the diagnosis and treatment of retinal diseases.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.16609","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"baochenfu","organizationType":"community","sourceUrl":"https://github.com/baochenfu/OCT-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_jor-bench_97431ead","familyId":"bmf_872a8906633f","name":"JOR-Bench","oneLine":"JOR-Bench is a collection of five Japanese-language benchmarks for LLMs in operations research, covering 1,319 problems from IndustryOR, MAMO, NL4OPT, OptiBench, and OptMATH, with pairs of Japanese problem statements and numerical answers.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-18","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.16777","pdf":"https://arxiv.org/pdf/2607.16777","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.16777"},"evidence":{"snippet":"We present JOR-Bench, a collection of five Japanese-language benchmarks for evaluating the ability of large language models (LLMs) to formulate and solve operations research (OR) problems.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.16777"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"JOR-Bench is a collection of five Japanese-language benchmarks for LLMs in operations research, covering 1,319 problems from IndustryOR, MAMO, NL4OPT, OptiBench, and OptMATH, with pairs of Japanese problem statements and numerical answers.","whyItMatters":"Provides a standardized Japanese-language evaluation for OR formulation and solving, enabling cross-lingual comparison and highlighting language-specific issues.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bb4f2158b0200316b5162cf22af86bd22847fe5bd9bb282d9a99e947905e267e"},"motivation":"We present JOR-Bench, a collection of five Japanese-language benchmarks for evaluating the ability of large language models (LLMs) to formulate and solve operations research (OR) problems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.16777","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"The authors","organizationType":"community","sourceUrl":"https://arxiv.org/abs/2607.16777","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_jarvisbench_2366ad17","familyId":"bmf_4e23273b5c08","name":"JarvisBench","oneLine":"JarvisBench measures mediation in long-horizon agent workflows with two tracks: agent-collaboration and user-interaction. Built on WildClaw tasks and a reference Jarvis prototype.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.16610","pdf":"https://arxiv.org/pdf/2607.16610","project":"https://cchen1436.github.io/jarvis","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.16610"},"evidence":{"snippet":"In this work, we introduce JarvisBench, a benchmark for measuring the dual value of mediation in long-horizon agent workflows.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.16610"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"JarvisBench measures mediation in long-horizon agent workflows with two tracks: agent-collaboration and user-interaction. Built on WildClaw tasks and a reference Jarvis prototype.","whyItMatters":"Could fill a gap in evaluating agent-user interaction, but unclear path for reuse.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"14bc45d8d26c66fd5d6d72e756157a1a41832f265b96b59b70f9eb3c49ca135f"},"motivation":"Long-horizon AI agents are becoming increasingly capable, yet their interaction with users remains surprisingly thin.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.16610","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_closer-bench_7847c9e5","familyId":"bmf_01919ee4d7a3","name":"CLOSER-Bench","oneLine":"CLOSER-Bench evaluates budgeted cross-stage design closure for hardware agents with spec-to-RTL, RTL-to-GDS, and spec-to-GDS tasks, using open-source tools and recording quality, progress, tool cost, and recovery.","area":"Code & Software","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Semiconductors","Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.16632","pdf":"https://arxiv.org/pdf/2607.16632","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.16632"},"evidence":{"snippet":"We introduce CLOSER-Bench, a controlled evaluation protocol for budgeted cross-stage design closure.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.16632"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CLOSER-Bench evaluates budgeted cross-stage design closure for hardware agents with spec-to-RTL, RTL-to-GDS, and spec-to-GDS tasks, using open-source tools and recording quality, progress, tool cost, and recovery.","whyItMatters":"Provides a controlled protocol for hardware design closure, addressing the gap in evaluating agents across abstraction boundaries.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"acbab26b48fbb48e1c0fbb3cd0dbdb9c9c75271206a0fe522f03dc30e0d2dd18"},"motivation":"Hardware engineering exposes coding agents to a form of long-horizon work that is difficult to capture with pass-at-k: progress is continuous, tool feedback is delayed and heterogeneous, and a backend failure may require revising RTL rather than tuning another physical-design parameter.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.16632","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"The authors","organizationType":"community","sourceUrl":"https://arxiv.org/abs/2607.16632","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_stabilitybench_c271b529","familyId":"bmf_56ebe7cdf845","name":"StabilityBench","oneLine":"StabilityBench is a benchmark operator that converts single-turn benchmark queries into multi-turn interaction histories with injected user simulations, evaluating LLM performance stability across demographic proxies and sycophantic baits on existing benchmarks.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20558","pdf":"https://arxiv.org/pdf/2607.20558","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20558"},"evidence":{"snippet":"We propose StabilityBench, a principled, general and model-agnostic benchmark operator that transforms single-turn benchmark queries into multi-turn interaction histories.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20558"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"StabilityBench is a benchmark operator that converts single-turn benchmark queries into multi-turn interaction histories with injected user simulations, evaluating LLM performance stability across demographic proxies and sycophantic baits on existing benchmarks.","whyItMatters":"Static benchmarks may not capture real-world conversational variability; StabilityBench highlights performance instability under realistic multi-turn conditions, motivating more realistic evaluation settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0b80505a27112ab586568270cfdea41ea236860ba0814c7d7f75c69bb8f682c0"},"motivation":"AI Assistants are increasingly deployed in high-stakes settings, such as healthcare or government services.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20558","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_slapbench_99d61c1b","familyId":"bmf_d255d1cf42d2","name":"SLAPBench","oneLine":"SLAPBench evaluates multimodal LLMs on four-finger SLAP fingerprint verification using NIST SD302b, with 7,832 image pairs. It assesses identity verification under zero-shot, task-description, and similarity-scoring prompts, reporting metrics like AUC and FAR.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.15517","pdf":"https://arxiv.org/pdf/2607.15517","project":null,"code":"https://github.com/bibeshpyakurel/SLAPBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.15517"},"evidence":{"snippet":"We introduce SLAPBench, the first benchmark for MLLM-based four-finger SLAP fingerprint verification, built from NIST SD302b with 7,832 pairs (176 mated, 7,656 non-mated).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.15517"},"ranking":{"90d":{"score":29,"rank":365,"coverage":0.55,"confidence":"Low"}},"description":"SLAPBench evaluates multimodal LLMs on four-finger SLAP fingerprint verification using NIST SD302b, with 7,832 image pairs. It assesses identity verification under zero-shot, task-description, and similarity-scoring prompts, reporting metrics like AUC and FAR.","whyItMatters":"First benchmark for MLLM-based SLAP fingerprint verification, enabling systematic comparison of models and prompting strategies for biometric identity verification tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5525872300f07a8a02b953c4bd78003e244a158b531bc099345a0ba50035a390"},"motivation":"Four-finger SLAP fingerprints are flat live-scan impressions of the index, middle, ring, and little fingers of one hand, used for identity verification in border control and law enforcement.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.15517","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_seq2synth_0ff26fb5","familyId":"bmf_a0760e0a5ca8","name":"Seq2Synth","oneLine":"Seq2Synth evaluates temporal fidelity of synthetic sequential tabular data across timestamp, cross-sectional, longitudinal, structural, and privacy dimensions, using a taxonomy to determine applicable metrics. It spans seven core datasets and multiple generators.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-07-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.15606","pdf":"https://arxiv.org/pdf/2607.15606","project":null,"code":"https://github.com/KiwanKwon/Seq2Synth","data":null,"hfPaper":"https://huggingface.co/papers/2607.15606"},"evidence":{"snippet":"We introduce Seq2Synth, a taxonomy-guided benchmark for evaluating whether synthetic sequential tabular data preserve these temporal structures.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.15606"},"ranking":{"90d":{"score":39,"rank":287,"coverage":0.55,"confidence":"Low"}},"description":"Seq2Synth evaluates temporal fidelity of synthetic sequential tabular data across timestamp, cross-sectional, longitudinal, structural, and privacy dimensions, using a taxonomy to determine applicable metrics. It spans seven core datasets and multiple generators.","whyItMatters":"Addresses the gap in evaluating temporal structure in synthetic tabular data, which static metrics miss, providing a standardized framework for researchers and practitioners.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2f587e57b042dc02da5ef87bc2b3c9955aec5c2e11ab2aed80564ecb083fa170"},"motivation":"Synthetic sequential tabular data are increasingly used for privacy-preserving data sharing and data-driven research, but evaluating their fidelity remains difficult because temporal structure is easily lost under conventional tabular metrics.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.15606","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_paraguibench_bf12bca6","familyId":"bmf_f889462186a8","name":"ParaGUIBench","oneLine":"ParaGUIBench aims to benchmark parallel execution and coordination of multiple GUI agents across separate desktop instances, with a dataset of 233 tasks and efficiency metrics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.22689","pdf":"https://arxiv.org/pdf/2607.22689","project":null,"code":"https://github.com/pkgunboat/ParaGUIBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.22689"},"evidence":{"snippet":"To close this gap, we introduce ParaGUIBench, to our knowledge, the first benchmark dedicated to parallel execution and coordination of multiple GUI agents on separate desktop instances.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.22689"},"ranking":{"90d":{"score":53,"rank":175,"coverage":0.55,"confidence":"Low"}},"description":"ParaGUIBench aims to benchmark parallel execution and coordination of multiple GUI agents across separate desktop instances, with a dataset of 233 tasks and efficiency metrics.","whyItMatters":"Could enable evaluation of parallel GUI agent coordination, potentially improving efficiency and success on long-horizon tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"db766832b3fcbdc4f443aec96d5d0666775212d8136c05a04af97e35869b8a4c"},"motivation":"Graphical user interface (GUI) agents are systems powered by large multimodal models (LMMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.22689","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_meventbench_dad2f997","familyId":"bmf_bdb6464af549","name":"MEventBench","oneLine":"MEventBench is a multi-event long video understanding benchmark introduced alongside the MoD-VLLM framework, evaluating temporal grounding and semantic understanding.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.15778","pdf":"https://arxiv.org/pdf/2607.15778","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.15778"},"evidence":{"snippet":"Moreover, we propose MEventBench, a challenging Multi-Event Long Video Benchmark for complex long video reasoning.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.15778"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MEventBench is a multi-event long video understanding benchmark introduced alongside the MoD-VLLM framework, evaluating temporal grounding and semantic understanding.","whyItMatters":"Could support evaluation of long-video understanding, but primarily serves as a testbed for the proposed method.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4446152766221e78ac06b96b8455613f180321d81d48c3bbfd04420ee9e51804"},"motivation":"Video Large Language Models (Video LLMs) have made significant advancements in various video understanding tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"2026 IEEE International Conference on Multimedia and Expo (ICME 2026)","evidence":"Accepted by 2026 IEEE International Conference on Multimedia and Expo (ICME 2026)","evidenceUrl":"https://arxiv.org/abs/2607.15778","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"2026 IEEE International Conference on Multimedia and Expo (ICME 2026)","reviewStatus":"accepted","decisionRaw":"Accepted by 2026 IEEE International Conference on Multimedia and Expo (ICME 2026)","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.15778","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by 2026 IEEE International Conference on Multimedia and Expo (ICME 2026)","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mergemedbench_c8868c16","familyId":"bmf_2eaf0e9f9c36","name":"MergeMedBench","oneLine":"MergeMedBench evaluates model merging methods for medical LVLMs across eight imaging modalities, comprising 16 LoRA fine-tuned models. It provides evaluation datasets and released model checkpoints for benchmarking merging approaches.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.15661","pdf":"https://arxiv.org/pdf/2607.15661","project":null,"code":"https://github.com/MedAI-T/MergeMedBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.15661"},"evidence":{"snippet":"We introduce MergeMedBench, a comprehensive benchmark spanning eight imaging modalities and diverse clinical task types, comprising 16 LoRA fine-tuned models built upon two mainstream architectures.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.15661"},"ranking":{"90d":{"score":29,"rank":366,"coverage":0.55,"confidence":"Low"}},"description":"MergeMedBench evaluates model merging methods for medical LVLMs across eight imaging modalities, comprising 16 LoRA fine-tuned models. It provides evaluation datasets and released model checkpoints for benchmarking merging approaches.","whyItMatters":"Addresses the lack of systematic evaluation for model merging in medical imaging, enabling comparison of merging methods and serving as a practical baseline.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"666a80260bf52cd62c7378756cf128bd43fbe792867a4b80ab62254e3842d030"},"motivation":"Large vision-language models (LVLMs) can be adapted to specialized medical imaging tasks via parameter-efficient fine-tuning approaches such as low-rank adaptation (LoRA), leading to a growing ecosystem of expert models tailored to specific imaging modalities and clinical scenarios.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.15661","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_imbench_d224375e","familyId":"bmf_7d389515cfd2","name":"IMBench","oneLine":"IMBench evaluates intuitive robotic manipulation, combining perception, physical reasoning, action generation, and execution across 35 tasks and 14K trajectories.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Reasoning","Robot manipulation"],"topics":["Robotics","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.15641","pdf":"https://arxiv.org/pdf/2607.15641","project":"https://imbench.org/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.15641"},"evidence":{"snippet":"We introduce IMBENCH, a benchmark designed to evaluate intuitive manipulation as an integrated capability spanning perception, physical reasoning, action generation, and iterative execution.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.15641"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"IMBench evaluates intuitive robotic manipulation, combining perception, physical reasoning, action generation, and execution across 35 tasks and 14K trajectories.","whyItMatters":"Could provide a more holistic benchmark for robotic manipulation, but details on public access and evaluation protocol are unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d82ba7069530c355226e5fb73b20157e03a8315593502be67ffa3ab4fc0bc8a6"},"motivation":"Humans combine reasoning and motor control to solve complex manipulation tasks under diverse constraints.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"SemRob Workshop, RSS 2026","evidence":"Accepted to SemRob Workshop, RSS 2026. Project Website: https://imbench.org/","evidenceUrl":"https://arxiv.org/abs/2607.15641","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"SemRob Workshop, RSS 2026","reviewStatus":"accepted","decisionRaw":"Accepted to SemRob Workshop, RSS 2026. Project Website: https://imbench.org/","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.15641","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to SemRob Workshop, RSS 2026. Project Website: https://imbench.org/","level":"author-claim"}]}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_decodem_118f6692","familyId":"bmf_7c2b484ff2de","name":"DECODEM","oneLine":"DECODEM provides benchmark datasets for evaluating automated extraction of corporate governance variables from organizational documents, with human-annotated charters and bylaws.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.15879","pdf":"https://arxiv.org/pdf/2607.15879","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.15879"},"evidence":{"snippet":"This paper introduces DECODEM, a set of benchmark datasets for evaluating the automated extraction of corporate governance variables from organizational documents.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.15879"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DECODEM provides benchmark datasets for evaluating automated extraction of corporate governance variables from organizational documents, with human-annotated charters and bylaws.","whyItMatters":"Could support legal research by providing standardized datasets for evaluating extraction methods, but availability is not confirmed.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"80cf46647e89aab5ccb69368bd7f271446d9232a3e15ba051ec1fa73b416dbc8"},"motivation":"Much empirical legal research depends on translating unstructured text into structured variables.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.15879","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_continuitybench_4edb780a","familyId":"bmf_74fd5bbd7858","name":"ContinuityBench","oneLine":"ContinuityBench evaluates stateful failover in multi-provider LLM routing. It measures Continuity Preservation Rate (CPR) and Continuity Latency Overhead (CLO) using synthetic conversation graphs under injected provider failures, with an LLM-as-a-judge scoring context preservation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-17","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.15899","pdf":"https://arxiv.org/pdf/2607.15899","project":null,"code":"https://github.com/Vishal-sys-code/continuity-bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.15899"},"evidence":{"snippet":"Furthermore, we release continuity-bench, https://github.com/Vishal-sys-code/continuity-bench, an open evaluation harness designed to stress-test context preservation under high-concurrency provider failure conditions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.15899"},"ranking":{"90d":{"score":29,"rank":380,"coverage":0.7,"confidence":"Medium"}},"description":"ContinuityBench evaluates stateful failover in multi-provider LLM routing. It measures Continuity Preservation Rate (CPR) and Continuity Latency Overhead (CLO) using synthetic conversation graphs under injected provider failures, with an LLM-as-a-judge scoring context preservation.","whyItMatters":"Production LLM deployments rely on failover mechanisms that often lose conversational context, degrading user experience. This benchmark provides a standardized way to quantify and compare context preservation and latency trade-offs across different routing architectures.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e3ed0c74b4f0d991c7b651bbde0bf14acbce3836bf9e7b5e6ffd6db48b55bb9d"},"motivation":"In production large language model (LLM) deployments, high API availability guarantees do not equate to conversational continuity.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.15899","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_chronobench_44b7b76d","familyId":"bmf_384c2b00f08c","name":"ChronoBench","oneLine":"ChronoBench evaluates long-term temporal understanding in remote sensing across four progressive cognitive levels: land cover perception, temporal recognition, long-term memory, and spatio-temporal reasoning. It comprises 12 sub-tasks and 17,689 QA pairs over 3,469 images spanning 500 regions across 39 U.S. cities.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.15768","pdf":"https://arxiv.org/pdf/2607.15768","project":null,"code":"https://github.com/IntelliSensing/GeoChrono","data":null,"hfPaper":"https://huggingface.co/papers/2607.15768"},"evidence":{"snippet":"To fill this gap, we introduce ChronoBench, a multidimensional benchmark that decomposes this task into four progressive cognitive levels (i.e., Land Cover Perception, Temporal Recognition, Long-Term Memory, and Spatio-Temporal Reasoning).","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":9,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.15768"},"ranking":{"90d":{"score":57,"rank":150,"coverage":0.7,"confidence":"Medium"}},"description":"ChronoBench evaluates long-term temporal understanding in remote sensing across four progressive cognitive levels: land cover perception, temporal recognition, long-term memory, and spatio-temporal reasoning. It comprises 12 sub-tasks and 17,689 QA pairs over 3,469 images spanning 500 regions across 39 U.S. cities.","whyItMatters":"Existing remote sensing benchmarks typically focus on static or bi-temporal analysis, lacking a systematic dissection of long-term temporal competencies. ChronoBench provides a multidimensional evaluation that isolates specific cognitive bottlenecks, enabling targeted model improvement. Its integration with lmms-eval supports standardized comparison of multimodal LLMs for satellite image time series.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4209e15b06135587f7b1fe72cd1383fc1680fe72216c0a023a38d9dcacd2c4db"},"motivation":"Remote sensing offers an unparalleled vantage point for observing the Earth's long-term surface evolution, yet it demands that a model not only perceive land cover at isolated moments, but also track changes, memorize evolution histories, and reason across time and space.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ACM MM 2026","evidence":"Accepted to ACM MM 2026","evidenceUrl":"https://arxiv.org/abs/2607.15768","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ACM MM 2026","reviewStatus":"accepted","decisionRaw":"Accepted to ACM MM 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.15768","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to ACM MM 2026","level":"author-claim"}]}],"publishers":[{"name":"IntelliSensing Lab","organizationType":"academic-lab","sourceUrl":"https://github.com/IntelliSensing/GeoChrono","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ave-compass_477a3826","familyId":"bmf_7977347d6b84","name":"AVE-Compass","oneLine":"AVE-Compass evaluates audio-visual editing models on 145 source videos and 196 instructions with 2,688 checklist items, scoring Instruction Following, Fidelity Preserving, Realism, and Editing Intent via MLLM judging and automated metrics.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.MM"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.24821","pdf":"https://arxiv.org/pdf/2607.24821","project":null,"code":"https://github.com/NJU-LINK/AVE-Compass","data":null,"hfPaper":"https://huggingface.co/papers/2607.24821"},"evidence":{"snippet":"We introduce AVE-Compass, a comprehensive benchmark with 145 curated source videos, 196 audio-visually coupled editing instructions, and 2,688 fine-grained checklist items.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":18,"hfDailySubmittedAt":"2026-08-06T00:00:00.000Z","githubStars":8,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24821"},"ranking":{"90d":{"score":61,"rank":129,"coverage":0.7,"confidence":"Medium"}},"description":"AVE-Compass evaluates audio-visual editing models on 145 source videos and 196 instructions with 2,688 checklist items, scoring Instruction Following, Fidelity Preserving, Realism, and Editing Intent via MLLM judging and automated metrics.","whyItMatters":"This benchmark addresses the gap in evaluating coordinated audio-visual edits, providing a structured way to measure cross-modal consistency and non-target preservation, which is critical for advancing real-world video editing systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"40d4b0b484cd6f1adade0e713f9f134250f79ecdab3a5792daef44fac5d87f71"},"motivation":"While instruction-based video editing has advanced rapidly, real-world videos contain tightly coupled audio and visual signals, and editing one modality often requires coordinated changes in the other.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24821","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"NJU-LINK","organizationType":"academic-lab","sourceUrl":"https://github.com/NJU-LINK/AVE-Compass","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_viabench_7ade7668","familyId":"bmf_3bab85546b0e","name":"VIABench","oneLine":"VIABench evaluates multimodal large language models on three tasks from first-person videos of visually impaired individuals: proactive reminder, visual question answering, and vision-guided interaction. It includes 761 videos, 46.9 hours, and 14,526 annotations, with protocols for online and offline settings.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14660","pdf":"https://arxiv.org/pdf/2607.14660","project":null,"code":"https://github.com/MCG-NJU/VIABench","data":null,"hfPaper":"https://huggingface.co/papers/2607.14660"},"evidence":{"snippet":"To fill this gap, we introduce VIABench, a comprehensive video benchmark specifically designed to evaluate MLLMs in Visually Impaired Assistance scenarios using first-person videos recorded or shared by VIIs themselves.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":9,"hfDailySubmittedAt":"2026-07-17T00:00:00.000Z","githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14660"},"ranking":{"90d":{"score":55,"rank":171,"coverage":0.7,"confidence":"Medium"}},"description":"VIABench evaluates multimodal large language models on three tasks from first-person videos of visually impaired individuals: proactive reminder, visual question answering, and vision-guided interaction. It includes 761 videos, 46.9 hours, and 14,526 annotations, with protocols for online and offline settings.","whyItMatters":"General MLLMs are rarely tested for real-world assistance of blind users. VIABench focuses on tasks like anticipating navigation-critical events, which are underrepresented in existing benchmarks, providing a practical measure of model utility in assistive contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d1ad5b80f3052a709b1912c1c26133d7314ca31648473995fab20925fa140868"},"motivation":"Visually impaired individuals (VIIs) encounter significant daily challenges due to limited access to visual information.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14660","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MCG-NJU","organizationType":"academic-lab","sourceUrl":"https://github.com/MCG-NJU/VIABench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_the-energy-society_dce5cbc5","familyId":"bmf_ac7dd1dd7b14","name":"The Energy Society","oneLine":"The Energy Society is a multi-agent simulation environment where LLM agents operate under energy constraints tied to token generation, completing jobs and donating energy to survive. The evaluation measures emergent cooperative and competitive behavior across different model sizes and incentive settings.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.14865","pdf":"https://arxiv.org/pdf/2607.14865","project":null,"code":"https://github.com/LucasBergholdt/EnergySociety","data":null,"hfPaper":"https://huggingface.co/papers/2607.14865"},"evidence":{"snippet":"The Energy Society is a compact testbed for studying the interaction between token costs and group incentives under a survival pressure.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14865"},"ranking":{"90d":{"score":39,"rank":288,"coverage":0.55,"confidence":"Low"}},"description":"The Energy Society is a multi-agent simulation environment where LLM agents operate under energy constraints tied to token generation, completing jobs and donating energy to survive. The evaluation measures emergent cooperative and competitive behavior across different model sizes and incentive settings.","whyItMatters":"The environment addresses the gap in studying how energy costs and survival pressures shape agent cooperation and competition, providing a testbed for evaluating multi-agent systems in resource-constrained settings. It could support decisions on agent design for sustainable and cooperative AI systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e4caeb06b30d2dd8147a970256106bc38c4eaf237a4f8b34b837194950d1762f"},"motivation":"LLM-based agents are increasingly deployed in multi-agent environments whose incentives can shape their behavior.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"AITC 2026","evidence":"Accepted at AITC 2026","evidenceUrl":"https://arxiv.org/abs/2607.14865","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"AITC 2026","reviewStatus":"accepted","decisionRaw":"Accepted at AITC 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.14865","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at AITC 2026","level":"author-claim"}]}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_symbalbench_84a0aab6","familyId":"bmf_26aed8f715b6","name":"SymbalBench","oneLine":"SymbalBench evaluates automated detection of systematic misalignments in MLLM-generated image captions. It comprises 420 vision-language datasets (1.7 million image-text pairs) from natural and medical domains, each annotated with known systematic misalignments.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.15216","pdf":"https://arxiv.org/pdf/2607.15216","project":null,"code":"https://github.com/Stanford-AIMI/Symbal","data":null,"hfPaper":"https://huggingface.co/papers/2607.15216"},"evidence":{"snippet":"As our second key contribution, we introduce SymbalBench, a benchmark designed to evaluate automated methods on our proposed task.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.15216"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SymbalBench evaluates automated detection of systematic misalignments in MLLM-generated image captions. It comprises 420 vision-language datasets (1.7 million image-text pairs) from natural and medical domains, each annotated with known systematic misalignments.","whyItMatters":"MLLM-generated captions often contain recurring errors tied to visual features, which can degrade downstream tasks. SymbalBench provides a standardized testbed to assess methods for surfacing such systematic captioning failures, aiding dataset auditing and model improvement.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0718cfa0851993fa77465aa34998a9780399ae8aa865c14932469950b3c6d10a"},"motivation":"Multimodal large language models (MLLMs) often introduce errors when generating image captions, resulting in misaligned image-text pairs.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.15216","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Stanford AIMI","organizationType":"academic-lab","sourceUrl":"https://github.com/Stanford-AIMI/Symbal","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_saferelbench_483807a4","familyId":"bmf_e7af30443a9d","name":"SafeRelBench","oneLine":"SafeRelBench evaluates VLM-driven embodied agents on household tasks with process-level safety constraints, focusing on spatial relations such as support, containment, and proximity. It includes 507 executable samples (248 spatial-relation, 259 control) and measures whether agents satisfy safety conditions before risk-prone actions, alongside task success.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14543","pdf":"https://arxiv.org/pdf/2607.14543","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.14543"},"evidence":{"snippet":"To address this gap, we introduce SAFERELBENCH, a spatial-relation-aware safety benchmark with 507 executable evaluation samples, including 248 spatial-relation samples and 259 non-spatial control samples.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14543"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SafeRelBench evaluates VLM-driven embodied agents on household tasks with process-level safety constraints, focusing on spatial relations such as support, containment, and proximity. It includes 507 executable samples (248 spatial-relation, 259 control) and measures whether agents satisfy safety conditions before risk-prone actions, alongside task success.","whyItMatters":"Safety in embodied agents depends on spatial awareness during action sequences, not just final outcomes. SafeRelBench fills a gap by quantifying process-level safety compliance, enabling comparison across agents and highlighting the need for improved spatial reasoning in safe planning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"90b9a8458215916985e3beec8e3cda0da201989b6161f7a78a6812b7884e99e6"},"motivation":"Vision-language models (VLMs) are increasingly used as the reasoning backbone of embodied agents, enabling robots to interpret visual scenes, follow language instructions, and plan multi-step actions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14543","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Authors of SafeRelBench","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.14543","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_rw-voice-eq-bench_67940c5c","familyId":"bmf_19d351184c25","name":"RW-Voice-EQ Bench","oneLine":"The Real World Voice EQ Bench evaluates voice AI systems across TTS, STS, SU, and ASR, focusing on how well models use acoustic information beyond text. It assesses dimensions like naturalness, expressiveness, identity stability, reliability, vocal affect use, and robustness to real-world conditions such as accent, emotion, noise, and conversation.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SD"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14846","pdf":"https://arxiv.org/pdf/2607.14846","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.14846"},"evidence":{"snippet":"To this end, we introduce the Real World Voice EQ Bench, a multidimensional benchmark for evaluating voice AI across text-to-speech (TTS), speech-to-speech (STS), speech understanding (SU), and automatic speech recognition (ASR).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":11,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14846"},"ranking":{"90d":{"score":54,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"The Real World Voice EQ Bench evaluates voice AI systems across TTS, STS, SU, and ASR, focusing on how well models use acoustic information beyond text. It assesses dimensions like naturalness, expressiveness, identity stability, reliability, vocal affect use, and robustness to real-world conditions such as accent, emotion, noise, and conversation.","whyItMatters":"Current voice AI benchmarks often evaluate isolated capabilities like word error rate or text-based dialogue quality, missing how systems harness acoustic information central to spoken language. This benchmark highlights that performance varies across dimensions, showing that a single aggregate score is insufficient and that real-world conditions expose failures not captured by clean-speech tests. It supports more nuanced evaluation and improvement of voice AI systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ab5af34bf14d8c9c9d952b5298f5eb8f486506392ed9d8dc6ac6f83dbcc56f99"},"motivation":"Current voice AI benchmarks typically evaluate isolated capabilities such as speech intelligibility, word error rate, or text-based dialogue quality, but they rarely test whether systems harness the acoustic information that distinguishes spoken language from its textual representation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14846","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_omniabench_e9e6c4ef","familyId":"bmf_6ab9ee1bd426","name":"OmniaBench","oneLine":"The evaluation object is unclear from the provided information.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14989","pdf":"https://arxiv.org/pdf/2607.14989","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.14989"},"evidence":{"snippet":"We introduce OmniaBench, a benchmark for evaluating general agents across diverse scenarios with explicit state spaces.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14989"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"The evaluation object is unclear from the provided information.","whyItMatters":"The evaluation gap and practical decision value are unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7321f5df32355b8eef13cea8752328e05bfff2efcd49f269f6958cbab05ea239"},"motivation":"Large language models are increasingly evolving from text generators into general agents capable of understanding user requests, invoking external tools, and completing complex tasks through interaction.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14989","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_medfailbench_90f26faa","familyId":"bmf_539cc8579836","name":"MedFailBench","oneLine":"The evaluation object is unclear from the provided information.","area":"Safety & Trustworthiness","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Safety"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.15166","pdf":"https://arxiv.org/pdf/2607.15166","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.15166"},"evidence":{"snippet":"We present a synthetic benchmark and failure atlas built by a clinician.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.15166"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"The evaluation object is unclear from the provided information.","whyItMatters":"The evaluation gap and practical decision value are unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d16bb94e520321bc9eeeedb4468346730904d1df61f48a1b5bec2c0246ef335e"},"motivation":"Most medical AI benchmarks measure whether a model knows the correct answer.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.15166","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_mcpevol-bench_3df86292","familyId":"bmf_dfd5c20c84be","name":"MCPEvol-Bench","oneLine":"The evaluation object is unclear from the provided information.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14642","pdf":"https://arxiv.org/pdf/2607.14642","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.14642"},"evidence":{"snippet":"To bridge this gap, we introduce \\textbf{MCPEvol-Bench}, a novel benchmark for evaluating the task-solving capabilities of LLM agents under dynamic toolset evolution.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14642"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"The evaluation object is unclear from the provided information.","whyItMatters":"The evaluation gap and practical decision value are unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"30cc1765ad8772c9749b1a33ed33fb2eeb23e606abae7c9278f663596aa577bd"},"motivation":"As Model Context Protocol (MCP) servers emerge as the core infrastructure for connecting LLMs with external tools, existing benchmarks leverage real-world MCP servers to evaluate LLM agents' tool-using capabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14642","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_drawingvqa_70f98873","familyId":"bmf_869e9acc7076","name":"DrawingVQA","oneLine":"The evaluation object is unclear from the provided information.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.15418","pdf":"https://arxiv.org/pdf/2607.15418","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.15418"},"evidence":{"snippet":"We introduce DrawingVQA, the first benchmark designed to evaluate multimodal large language models (MLLMs) on real-world construction drawings -- a core media in architecture, civil, and many other engineering practices.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.15418"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"The evaluation object is unclear from the provided information.","whyItMatters":"The evaluation gap and practical decision value are unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"df73c25747555150e087c24786463c9155a6c10100f2f7b19dc88f17c4c0701c"},"motivation":"We introduce DrawingVQA, the first benchmark designed to evaluate multimodal large language models (MLLMs) on real-world construction drawings -- a core media in architecture, civil, and many other engineering practices.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"paper","evidence":"CVPR 2026 Findings accepted paper","evidenceUrl":"https://arxiv.org/abs/2607.15418","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"paper","reviewStatus":"accepted","decisionRaw":"CVPR 2026 Findings accepted paper","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.15418","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"CVPR 2026 Findings accepted paper","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_chronoqg_6a4613e9","familyId":"bmf_ac031db3c5df","name":"ChronoQG","oneLine":"The evaluation object is unclear from the provided information.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14770","pdf":"https://arxiv.org/pdf/2607.14770","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.14770"},"evidence":{"snippet":"We propose ChronoQG, the first temporally expressive and hop-bounded benchmark construction framework for TKGQG.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14770"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"The evaluation object is unclear from the provided information.","whyItMatters":"The evaluation gap and practical decision value are unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"74860b5074cf170a6ce0bc75155bf3358e70cf387aae5becf8c8ce75ce799307"},"motivation":"Knowledge graph question generation (KGQG) aims to generate natural-language questions from structured graph evidence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14770","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cfm-bench_fc3a1cd0","familyId":"bmf_41e6fe8357e1","name":"CFM-Bench","oneLine":"The evaluation object is unclear from the provided information.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14975","pdf":"https://arxiv.org/pdf/2607.14975","project":"https://www.chaspark.com/\\#/s/CFM-Bench","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.14975"},"evidence":{"snippet":"We release CFM-Bench, a unified multi-domain, multi-task benchmark comprising 157,900 official single-frame examples from six domains spanning 3GPP statistical simulation, two ray-tracing pipelines, terrestrial and aerial measurements, and synchronized vehicular multimodal simulation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14975"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"The evaluation object is unclear from the provided information.","whyItMatters":"The evaluation gap and practical decision value are unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f750cc9655041d308723f72a1ecd0a15296222f7715a5026bf722812bc88441b"},"motivation":"Channel foundation models (CFMs) are commonly evaluated in model-specific pipelines that differ in data, radio configurations, partitions, adaptation procedures, task definitions, and metrics, preventing reproducible comparison across CFMs and against task-specific networks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14975","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_atrex-bench_ab05aff2","familyId":"bmf_068d0e6d2ab8","name":"Atrex-Bench","oneLine":"Evaluates coding agents on generating GPU kernels from PyTorch references across 30 operators and 440 shapes derived from production inference traces. Scoring uses a three-stage evaluator measuring compile success, numerical correctness, and speed-of-light (SOL) efficiency against a cached roofline. The benchmark includes 4 DSL backends and supports multi-vendor GPUs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Agents"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.14541","pdf":"https://arxiv.org/pdf/2607.14541","project":null,"code":"https://github.com/alibaba/atrex-bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.14541"},"evidence":{"snippet":"We present Atrex-Bench, a benchmark whose 30 operators and 440 shapes are sampled directly from full-cluster production inference traces of compute-limited, memory-rich GPUs.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":24,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14541"},"ranking":{"90d":{"score":61,"rank":130,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates coding agents on generating GPU kernels from PyTorch references across 30 operators and 440 shapes derived from production inference traces. Scoring uses a three-stage evaluator measuring compile success, numerical correctness, and speed-of-light (SOL) efficiency against a cached roofline. The benchmark includes 4 DSL backends and supports multi-vendor GPUs.","whyItMatters":"Prior GPU kernel benchmarks draw from synthetic or curated sources that diverge from deployed workloads. This benchmark provides a production-trace-driven evaluation that emphasizes operators consuming the most serving time, enabling assessment of agent performance on relevant tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"63e87ad19eac0773073fb3036c263af57bac1f75c8689e70d1acf14ca0e070ce"},"motivation":"Existing GPU kernel generation benchmarks draw problems from synthetic or curated sources that diverge from deployed workloads.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14541","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Alibaba","organizationType":"company-research-lab","sourceUrl":"https://github.com/alibaba/atrex-bench","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_alipay-pibench_aa13d775","familyId":"bmf_09cfe17ab4d7","name":"Alipay-PIBench","oneLine":"The evaluation object is unclear from the provided information.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14573","pdf":"https://arxiv.org/pdf/2607.14573","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.14573"},"evidence":{"snippet":"We introduce Alipay-PIBench, a benchmark for evaluating coding agents on realistic Alipay payment integration.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14573"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"The evaluation object is unclear from the provided information.","whyItMatters":"The evaluation gap and practical decision value are unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a66589c396fc03e7876fd0c387105fdd554bea2e8c6690ac53b21ae5a722378f"},"motivation":"Payment integration is a demanding repository-level software task: agents must select a suitable product, implement coordinated client-server flows, verify payment outcomes, and preserve consistency between transaction and business states.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14573","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_ae-uav_5141a634","familyId":"bmf_df940847cd49","name":"AE-UAV","oneLine":"AE-UAV is an air-to-air event-based UAV tracking benchmark with 178 flight sequences and continuous-time cubic B-spline annotations, supporting evaluation at arbitrary temporal resolutions. It includes multimodal auxiliary data and predefined train/validation/test splits.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14726","pdf":"https://arxiv.org/pdf/2607.14726","project":null,"code":"https://github.com/MSP-xEN/AE-UAV","data":null,"hfPaper":"https://huggingface.co/papers/2607.14726"},"evidence":{"snippet":"To bridge these gaps, we introduce AE-UAV, an air-to-air event-based UAV tracking benchmark.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":12,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14726"},"ranking":{"90d":{"score":56,"rank":163,"coverage":0.7,"confidence":"Medium"}},"description":"AE-UAV is an air-to-air event-based UAV tracking benchmark with 178 flight sequences and continuous-time cubic B-spline annotations, supporting evaluation at arbitrary temporal resolutions. It includes multimodal auxiliary data and predefined train/validation/test splits.","whyItMatters":"This benchmark addresses the lack of dedicated event-based datasets for air-to-air UAV tracking, enabling consistent evaluation and comparison of tracking methods under diverse motion and illumination conditions. It provides a public resource for developing and validating real-time tracking solutions on resource-constrained platforms.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1b901fc1cfdb4aa8c50e87d37739d2b4c2123a033e310031c27f0f265213ba6c"},"motivation":"Air-to-air (A2A) unmanned aerial vehicle (UAV) tracking is fundamental to airborne remote sensing of low-altitude aerial targets.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14726","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AE-UAV Team","organizationType":"academic-lab","sourceUrl":"https://github.com/MSP-xEN/AE-UAV","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_uesf-bench_80fcb44c","familyId":"bmf_d538adf3f888","name":"UESF-Bench","oneLine":"UESF-Bench evaluates embodied agents on unified language-guided human seeking and following in dynamic environments, covering semantic-guided exploration, behavior switching, and identity grounding across single- and multi-person settings. Scoring uses success metrics for both phases.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.13621","pdf":"https://arxiv.org/pdf/2607.13621","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.13621"},"evidence":{"snippet":"To address these limitations, we introduce the Unified Embodied Seeking and Following Benchmark (UESF-Bench), a large-scale and diverse benchmark for embodied human seeking and following.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.13621"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"UESF-Bench evaluates embodied agents on unified language-guided human seeking and following in dynamic environments, covering semantic-guided exploration, behavior switching, and identity grounding across single- and multi-person settings. Scoring uses success metrics for both phases.","whyItMatters":"Existing benchmarks assume the target is visible at start, missing realistic scenarios where agents must first find and then follow. UESF-Bench provides a unified evaluation to advance embodied agents in more practical human-robot interaction.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"72f1b5c64cec3d5a46ad1f6d58b752fbf886ec8273d3a3fde926a5b9df17e7d2"},"motivation":"Language-guided human following is an important capability for embodied agents, but existing benchmarks typically assume that the target person is visible at the start of an episode.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.13621","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_toolalignbench_7a0c33d1","familyId":"bmf_95022812d16e","name":"ToolAlignBench","oneLine":"The evaluation object is a set of 128 scenarios across 16 domains for tool-calling LLM agents in regulated industries, assessing conflicts between safety-aligned values and deployment instructions. The task involves processing confidential documents and measuring override behavior.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-15","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.14285","pdf":"https://arxiv.org/pdf/2607.14285","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.14285"},"evidence":{"snippet":"To empirically verify this phenomenon, we build a benchmark of 128 scenarios across 16 domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14285"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"The evaluation object is a set of 128 scenarios across 16 domains for tool-calling LLM agents in regulated industries, assessing conflicts between safety-aligned values and deployment instructions. The task involves processing confidential documents and measuring override behavior.","whyItMatters":"The evaluation gap is the lack of tests for conflicting value systems in agentic tool use. Practical value lies in identifying liability risks and tuning alignment strategies for regulated deployments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6bad161e55b7b44e9985b91f2d6762d5b3e449566835eeffc74e38c7ee1bbe6d"},"motivation":"Safety alignment in LLMs aims to align models with human values, but which values take precedence when they conflict?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"Pluralistic Alignment Workshop at ICML 2026","evidence":"Accepted to the Pluralistic Alignment Workshop at ICML 2026","evidenceUrl":"https://arxiv.org/abs/2607.14285","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"Pluralistic Alignment Workshop at ICML 2026","reviewStatus":"accepted","decisionRaw":"Accepted to the Pluralistic Alignment Workshop at ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.14285","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to the Pluralistic Alignment Workshop at ICML 2026","level":"author-claim"}]}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_real-bench_906823b4","familyId":"bmf_f0848f948867","name":"REAL-Bench","oneLine":"REAL-Bench evaluates vision-driven embodied agents in open-world mobile manipulation across 241 tasks spanning active exploration, visual distraction, articulated manipulation, and interactive disambiguation. The benchmark provides standardized task definitions and a simulator-based evaluation protocol.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.13653","pdf":"https://arxiv.org/pdf/2607.13653","project":null,"code":"https://github.com/InternRobotics/REAL","data":null,"hfPaper":"https://huggingface.co/papers/2607.13653"},"evidence":{"snippet":"To comprehensively evaluate this approach, we introduce REAL-Bench, a benchmark spanning 241 tasks across active exploration, visual distraction, articulated manipulation, and interactive disambiguation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":36,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.13653"},"ranking":{"90d":{"score":63,"rank":111,"coverage":0.7,"confidence":"Medium"}},"description":"REAL-Bench evaluates vision-driven embodied agents in open-world mobile manipulation across 241 tasks spanning active exploration, visual distraction, articulated manipulation, and interactive disambiguation. The benchmark provides standardized task definitions and a simulator-based evaluation protocol.","whyItMatters":"The benchmark targets the gap between simulation and real-world deployment for embodied agents, offering a repeatable evaluation for long-horizon tasks requiring visual grounding and interactive intent disambiguation. It enables systematic comparison of agent frameworks and informs progress toward practical deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d87a8843dd9c4628b22e06c66c59b9940846649238afd448749ac5ba5b990da7"},"motivation":"Real-world deployment of embodied agents requires active exploration, visual grounding, and interactive intent disambiguation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026","evidence":"Accepted to ECCV 2026. 57 pages. Code available at https://github.com/InternRobotics/REAL","evidenceUrl":"https://arxiv.org/abs/2607.13653","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026","reviewStatus":"accepted","decisionRaw":"Accepted to ECCV 2026. 57 pages. Code available at https://github.com/InternRobotics/REAL","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.13653","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to ECCV 2026. 57 pages. Code available at https://github.com/InternRobotics/REAL","level":"author-claim"}]}],"publishers":[{"name":"InternRobotics","organizationType":"academic-lab","sourceUrl":"https://github.com/InternRobotics/REAL","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_multiref-compass_396a4597","familyId":"bmf_83d3a7c119e7","name":"MultiRef-Compass","oneLine":"MultiRef-Compass evaluates multi-reference-to-audio-video generation systems on 350 curated samples. It assesses Basic Quality, Reference Consistency, Audio-Visual Consistency, and Instruction Following using 14 sub-metrics, combining automatic metrics with an MLLM-as-a-Judge framework.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14189","pdf":"https://arxiv.org/pdf/2607.14189","project":null,"code":"https://github.com/zxhhh0201/MultiRef-Compass","data":null,"hfPaper":"https://huggingface.co/papers/2607.14189"},"evidence":{"snippet":"To address this gap, we introduce MultiRef-Compass, a unified benchmark for MR2AV generation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":34,"hfDailySubmittedAt":"2026-07-17T00:00:00.000Z","githubStars":12,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14189"},"ranking":{"90d":{"score":66,"rank":96,"coverage":0.7,"confidence":"Medium"}},"description":"MultiRef-Compass evaluates multi-reference-to-audio-video generation systems on 350 curated samples. It assesses Basic Quality, Reference Consistency, Audio-Visual Consistency, and Instruction Following using 14 sub-metrics, combining automatic metrics with an MLLM-as-a-Judge framework.","whyItMatters":"Existing benchmarks focus on text-driven or single-reference generation and often ignore joint audio-video alignment. MultiRef-Compass addresses the gap by providing a public protocol for multi-reference composition, enabling reproducible comparison across models in a rapidly evolving generation task.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"56752da5440a76ba771efb3eed9a915e47f283e2c763da850ea795095b8f28a4"},"motivation":"Multi-reference-to-audio-video (MR2AV) generation aims to generate coherent audio-video content conditioned on multiple references and textual instructions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14189","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MultiRef-Compass Team","organizationType":"academic-lab","sourceUrl":"https://github.com/zxhhh0201/MultiRef-Compass","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mamabench_7ce134aa","familyId":"bmf_db4a560447e2","name":"MamaBench","oneLine":"MamaBench evaluates LLM robustness in maternal and child health diagnosis using counterfactual clinical narratives, with the Bias Trap Rate (BTR) metric.","area":"Safety & Trustworthiness","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Robustness"],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-07-15","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.14385","pdf":"https://arxiv.org/pdf/2607.14385","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.14385"},"evidence":{"snippet":"We introduce MamaBench, the first counterfactual benchmark for maternal and paediatric AI: 434 expert-authored clinical narratives in 217 pairs across 371 pathologies, evaluated via the Bias Trap Rate (BTR), the conditional probability that a model fails the counterfactual given success on the base case.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14385"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MamaBench evaluates LLM robustness in maternal and child health diagnosis using counterfactual clinical narratives, with the Bias Trap Rate (BTR) metric.","whyItMatters":"It highlights the gap between base accuracy and robust accuracy in clinical AI, showing that models can fail on clinically similar cases requiring different interventions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"79b8aad9e670b441115c9dcf37c6f69a3a9c3e37fc478b338a5de9d518cc2a19"},"motivation":"Large language models achieve strong scores on medical benchmarks, yet these benchmarks evaluate each question in isolation, providing no measure of whether a system can distinguish clinically similar presentations requiring different interventions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14385","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_keyframe-compass_c03623fb","familyId":"bmf_6d5f810b3b48","name":"KeyFrame-Compass","oneLine":"KeyFrame-Compass evaluates keyframe-conditioned video generation across 386 curated samples spanning three application domains, two video structures, two prompt granularities, two conditioning formats, and four keyframe densities. It jointly measures keyframe execution (presence, fidelity, temporal ordering, localization, persistence, uniqueness) and overall video quality via evidence-grounded MLLM judgments and specialized perception models.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14202","pdf":"https://arxiv.org/pdf/2607.14202","project":null,"code":"https://github.com/cactusqq/KeyFrame-Compass","data":null,"hfPaper":"https://huggingface.co/papers/2607.14202"},"evidence":{"snippet":"We present KeyFrame-Compass, the first comprehensive benchmark for evaluating keyframe-conditioned video generation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":42,"hfDailySubmittedAt":"2026-07-17T00:00:00.000Z","githubStars":10,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14202"},"ranking":{"90d":{"score":65,"rank":103,"coverage":0.7,"confidence":"Medium"}},"description":"KeyFrame-Compass evaluates keyframe-conditioned video generation across 386 curated samples spanning three application domains, two video structures, two prompt granularities, two conditioning formats, and four keyframe densities. It jointly measures keyframe execution (presence, fidelity, temporal ordering, localization, persistence, uniqueness) and overall video quality via evidence-grounded MLLM judgments and specialized perception models.","whyItMatters":"As keyframe-based workflows grow in video production, there is no standardized way to assess whether models faithfully reproduce prescribed keyframes while maintaining natural video quality. KeyFrame-Compass provides a controlled testbed that reveals trade-offs and degradation patterns, helping practitioners choose models based on constraint density and input format.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"df26a592cc7930a241131cbc5981ea9921587bfcbb0510b6b389ea433c5d1994"},"motivation":"Video generation increasingly relies on keyframe-based workflows, where creators specify a sequence of reference images to guide generation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14202","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"KeyFrame-Compass Team","organizationType":"academic-lab","sourceUrl":"https://github.com/cactusqq/KeyFrame-Compass","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_industrial-dexterity-benchmark_210a10b0","familyId":"bmf_35062a925b27","name":"Industrial Dexterity Benchmark","oneLine":"The Industrial Dexterity Benchmark (IDB) provides physical boards and tasks for industrial dexterous manipulation, including cable management, cable harness, and gearbox assembly. It evaluates a robot's ability to perform tasks such as cable insertion and grasping via end-to-end imitation learning policies.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics","Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14021","pdf":"https://arxiv.org/pdf/2607.14021","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.14021"},"evidence":{"snippet":"As a part of this work, we introduce three key contributions: a set of Industrial Dexterity Benchmark (IDB) boards aimed to mimic datacenter cable management, automotive cable harnesses, and gearbox assembly tasks; a scalable imitation learning framework (DAG-ROS); and a multimodal diffusion-based policy framework (AG-iDP3) that creates models fusing RGB images, point clouds, joint positions, and wrist-frame wrench data.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14021"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"The Industrial Dexterity Benchmark (IDB) provides physical boards and tasks for industrial dexterous manipulation, including cable management, cable harness, and gearbox assembly. It evaluates a robot's ability to perform tasks such as cable insertion and grasping via end-to-end imitation learning policies.","whyItMatters":"The benchmark addresses the lack of standardized evaluation for industrial dexterous manipulation, offering a repeatable protocol with defined tasks and success metrics. It enables systematic comparison of learning-based versus classical control methods, aiding automation decisions for high up-time industrial environments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2424e8b3066aae14fb49369473786dff7cbafad6b63ddc1bf338ec1d4ae6a5ca"},"motivation":"Dexterous manipulation remains a critical bottleneck in industrial automation; tasks such as cable routing, connector insertion, and precision assembly still rely heavily on manual labor despite decades of robotics research.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14021","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Industrial Dexterity Benchmark Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.14021","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_hypershadow_31f52aa8","familyId":"bmf_21616075ae21","name":"HyperShadow","oneLine":"HyperShadow evaluates binary classification of 3D point clouds as either native 3D objects or 3D projections of objects in 4-6 spatial dimensions, with static and temporal tracks, four corruption tiers, and fixed train/eval splits.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14419","pdf":"https://arxiv.org/pdf/2607.14419","project":null,"code":"https://github.com/AkshaySasi/hypershadow","data":"https://huggingface.co/datasets/AkshaySasi/hypershadow","hfPaper":"https://huggingface.co/papers/2607.14419"},"evidence":{"snippet":"We introduce HyperShadow, the first public benchmark in which the fourth, fifth, and sixth dimensions are spatial: the task is to decide whether a 3D point cloud is a native three-dimensional shape or the projection, the \"shadow\", of a rigid object living in R^N (N = 4-6).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":261,"hfDatasetLikes":1},"source":{"type":"arxiv","id":"2607.14419"},"ranking":{"90d":{"score":28,"rank":384,"coverage":1.0,"confidence":"High","datasetDownloadRank":21,"datasetRankPopulation":51}},"description":"HyperShadow evaluates binary classification of 3D point clouds as either native 3D objects or 3D projections of objects in 4-6 spatial dimensions, with static and temporal tracks, four corruption tiers, and fixed train/eval splits.","whyItMatters":"Addresses a gap in benchmarks for high-dimensional geometric data, providing a controlled testbed for studying projection signatures and out-of-distribution detection without physical reality claims.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0bc2b545df5e1b8bf15654455da6198b2e780d8904503c0da26df7ef632cdb80"},"motivation":"Machine-learning datasets labelled \"4D\" universally denote three spatial dimensions plus time.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14419","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AkshaySasi","organizationType":"community","sourceUrl":"https://github.com/AkshaySasi/hypershadow","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_gca-bench_71ae6cdc","familyId":"bmf_4566b765b276","name":"GCA-Bench","oneLine":"GCA-Bench evaluates robotic grasping in scenarios requiring scene-level reasoning and semantic constraints, comparing large foundation models on complex action tasks.","area":"Vision & 3D","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.14341","pdf":"https://arxiv.org/pdf/2607.14341","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.14341"},"evidence":{"snippet":"To address this gap, we propose GCA-Bench, a benchmark featuring challenging \\textit{grasping with complex action} scenarios that involve both scene-level reasoning and semantic constraints.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.14341"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GCA-Bench evaluates robotic grasping in scenarios requiring scene-level reasoning and semantic constraints, comparing large foundation models on complex action tasks.","whyItMatters":"The benchmark addresses a gap in grasping evaluation by moving beyond isolated visual pose detection to multi-step reasoning, offering a more realistic assessment for real-world robotic applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"901aa754e4aeb2d27f847ad43bae84f6e166e9977fa44754d2904f1f8793015d"},"motivation":"Robust robotic grasping remains a fundamental challenge for complex real-world applications.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.14341","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_devicesworld_1a9e0be1","familyId":"bmf_c77bf3b044f4","name":"DevicesWorld","oneLine":"DevicesWorld is an executable benchmark for cross-device agent evaluation, comprising 6,140 tasks across Android, Linux, and SmartHome environments, with rule-based verifiers for automatic scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.13465","pdf":"https://arxiv.org/pdf/2607.13465","project":null,"code":"https://github.com/AgenticOrgLab/DevicesWorld","data":null,"hfPaper":"https://huggingface.co/papers/2607.13465"},"evidence":{"snippet":"We introduce DevicesWorld, a large-scale executable benchmark for cross-device collaborative operation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.13465"},"ranking":{"90d":{"score":46,"rank":219,"coverage":0.55,"confidence":"Low"}},"description":"DevicesWorld is an executable benchmark for cross-device agent evaluation, comprising 6,140 tasks across Android, Linux, and SmartHome environments, with rule-based verifiers for automatic scoring.","whyItMatters":"Existing benchmarks focus on single-device environments, leaving a gap in evaluating agents that must coordinate across heterogeneous devices. DevicesWorld aims to address this by providing tasks with cross-device dependencies and automated verification, which could inform development of more capable multi-device agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bb7b831b85e4996ca18d28a1b11d156e22ee3aeca761fac80c556b3f321ae5b0"},"motivation":"LLM-based agents have rapidly improved at operating individual digital environments such as mobile applications, desktop systems, and smart homes.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.13465","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_vanillabench_f1c1ed7c","familyId":"bmf_4e12120315f2","name":"VanillaBench","oneLine":"VanillaBench evaluates the clean accuracy gap between adversarially trained models and vanilla (non-robust) reference models across four threat models. It defines a protocol for comparing robustness-accuracy trade-offs using model accuracy on standard benchmarks.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Robustness"],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.12545","pdf":"https://arxiv.org/pdf/2607.12545","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.12545"},"evidence":{"snippet":"We introduce VanillaBench, a systematic benchmark that makes this gap explicit.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.12545"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"VanillaBench evaluates the clean accuracy gap between adversarially trained models and vanilla (non-robust) reference models across four threat models. It defines a protocol for comparing robustness-accuracy trade-offs using model accuracy on standard benchmarks.","whyItMatters":"It addresses the evaluation gap in adversarial robustness research by quantifying the cost of robustness, providing practitioners with information needed to make informed deployment decisions regarding accuracy versus robustness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bdea16c79e9813caaf260f629be8ef2decf83a33ad50b0de7b4d2c9373bb8e39"},"motivation":"Adversarial robustness research has produced hundreds of defended models over the past decade, yet the literature almost universally reports robustness results in isolation: standard (clean) accuracy and adversarial accuracy of the robust model are shown, but the gap to the corresponding vanilla model is rarely quantified.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.12545","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_terralogic_3dd73c53","familyId":"bmf_5ec0cc94dc3b","name":"TerraLogic","oneLine":"TerraLogic evaluates hierarchical geospatial reasoning in Earth observation through 545 scenario-driven tasks spanning optical, SAR, and infrared imagery. Tasks include hazard vulnerability assessment, urban heat island analysis, and forest fragmentation dynamics. Evaluation uses tool-augmented agents with verifiable multi-step workflows and scored by step-wise and end-to-end metrics.","area":"Vision & 3D","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-07-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.12497","pdf":"https://arxiv.org/pdf/2607.12497","project":null,"code":"https://github.com/Ireliya/TerraLogic","data":null,"hfPaper":"https://huggingface.co/papers/2607.12497"},"evidence":{"snippet":"To address this gap, we introduce TerraLogic, a benchmark for geospatial reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":26,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.12497"},"ranking":{"90d":{"score":66,"rank":98,"coverage":0.55,"confidence":"Low"}},"description":"TerraLogic evaluates hierarchical geospatial reasoning in Earth observation through 545 scenario-driven tasks spanning optical, SAR, and infrared imagery. Tasks include hazard vulnerability assessment, urban heat island analysis, and forest fragmentation dynamics. Evaluation uses tool-augmented agents with verifiable multi-step workflows and scored by step-wise and end-to-end metrics.","whyItMatters":"Existing remote sensing benchmarks primarily target perception tasks, leaving a gap in assessing cognitive-level geospatial reasoning. TerraLogic provides a fixed dataset and protocol for comparing agent performance on compositional, long-horizon analysis, enabling systematic evaluation of tool-augmented reasoning across modalities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9139470f269706e477c14f4a154a97756138d19c3ecb1e3455aa4009b2bd61c2"},"motivation":"Beyond perception, reasoning is essential in remote sensing for advanced interpretation, inference, and decision-making.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.12497","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TerraLogic Team","organizationType":"academic-lab","sourceUrl":"https://github.com/Ireliya/TerraLogic","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_sis-bench_e31a9de3","familyId":"bmf_067ed93fb5e8","name":"SIS-Bench","oneLine":"SIS-Bench evaluates embodied spatial intelligence in UAV scenarios across two dimensions (spatial cognition and self-awareness) and three cognitive levels (perception, memory, reasoning), with 4,856 QA pairs from 1,646 real-world UAV videos.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.CV"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.12477","pdf":"https://arxiv.org/pdf/2607.12477","project":"https://choucisan.github.io/publications/self-in-space","code":"https://github.com/IntelliSensing/Self-in-Space","data":null,"hfPaper":"https://huggingface.co/papers/2607.12477"},"evidence":{"snippet":"To address this gap, we introduce SIS-Bench, a benchmark for evaluating embodied spatial intelligence in UAV scenarios under a unified self-in-space formulation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":11,"hfDailySubmittedAt":"2026-07-16T00:00:00.000Z","githubStars":31,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.12477"},"ranking":{"90d":{"score":71,"rank":68,"coverage":0.7,"confidence":"Medium"}},"description":"SIS-Bench evaluates embodied spatial intelligence in UAV scenarios across two dimensions (spatial cognition and self-awareness) and three cognitive levels (perception, memory, reasoning), with 4,856 QA pairs from 1,646 real-world UAV videos.","whyItMatters":"Existing UAV benchmarks are environment-centric, leaving agent self-awareness implicit. SIS-Bench provides a unified self-in-space evaluation protocol, revealing imbalances between spatial and self-related cognition and offering practical value for developing UAV embodied models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f1d33b45c10ac866cad55c10908802ca03020edc64a37af3818fb8104b29aca9"},"motivation":"Autonomous UAV systems increasingly rely on multimodal large language models (MLLMs) to operate in complex real-world environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.12477","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"IntelliSensing","organizationType":"academic-lab","sourceUrl":"https://github.com/IntelliSensing/Self-in-Space","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_prive-bench_ef1c909e","familyId":"bmf_cb1395d6b84b","name":"PriVE-Bench","oneLine":"PriVE-Bench evaluates vision-language models' visual grounding using paired original and counterfactual images, with PriVE-Tools extending to tool-derived evidence.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-14","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.16311","pdf":"https://arxiv.org/pdf/2607.16311","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.16311"},"evidence":{"snippet":"We introduce PriVE-Bench, a Prior-vs-Visual Evidence Benchmark that uses paired original and counterfactual images to distinguish visually grounded answers from prior-consistent errors.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.16311"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PriVE-Bench evaluates vision-language models' visual grounding using paired original and counterfactual images, with PriVE-Tools extending to tool-derived evidence.","whyItMatters":"Assesses whether vision-language models rely on learned priors rather than image content, and whether additional visual tools can improve grounding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fc31832d396eb7c618b0e2fa575e4b089072b1d2191f6326f06fd96b735efeba"},"motivation":"Vision-language models (VLMs) often answer visual questions using learned language and category priors rather than grounding their predictions in the image itself.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.16311","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_pm-bench_3de97f77","familyId":"bmf_ecbb460ae2fa","name":"PM-Bench","oneLine":"PM-Bench evaluates prospective memory in LLM agents through a text-based simulated seven-day week. Agents must maintain user intentions, execute delayed intentions, and monitor latent environment changes while performing ongoing activities.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.12385","pdf":"https://arxiv.org/pdf/2607.12385","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.12385"},"evidence":{"snippet":"We introduce PM-Bench, a text-based benchmark for measuring prospective memory capabilities in modern LLM agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.12385"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PM-Bench evaluates prospective memory in LLM agents through a text-based simulated seven-day week. Agents must maintain user intentions, execute delayed intentions, and monitor latent environment changes while performing ongoing activities.","whyItMatters":"PM-Bench fills a gap in evaluating agentic AI by measuring an understudied cognitive capability—prospective memory—in a controlled, repeatable setting. It provides a diagnostic tool for comparing LLM agents and guiding interventions to improve reliability in real-world tasks requiring memory for future actions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"57ee1bd17b05250289c0cbe599796abada7eb96ca182b1109ed88ec2a995a9e2"},"motivation":"A significant challenge in agentic AI is prospective memory: the ability to execute an intention at a specific future cue or state while other activities are ongoing.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.12385","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_memops_f8d37e23","familyId":"bmf_1d0b276100ed","name":"MemOps","oneLine":"MemOps evaluates conversational memory as a sequence of lifecycle operations (remembering, forgetting, updating, reflecting) with structured traces and six categories of operation-level probes, under adjacent-evidence and long-context settings.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.12893","pdf":"https://arxiv.org/pdf/2607.12893","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.12893"},"evidence":{"snippet":"We introduce MemOps, a benchmark that reformulates conversational memory as a sequence of lifecycle operations and represents each memory event with a structured trace specifying its trigger, target, scope, state transition, and supporting evidence.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.12893"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MemOps evaluates conversational memory as a sequence of lifecycle operations (remembering, forgetting, updating, reflecting) with structured traces and six categories of operation-level probes, under adjacent-evidence and long-context settings.","whyItMatters":"This benchmark addresses the gap in memory evaluation by providing operation-level diagnosis rather than final-answer accuracy, revealing specific failure modes in long-context, retrieval-based, parametric, and managed-memory systems, which is valuable for improving memory reliability in LLM agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"000d346270f70feb830ce9c035e05914550c4063c14ea44731362f56401b2ce4"},"motivation":"Long-term memory has become a foundational capability for LLM-based agents that accompany users across extended, multi-session interactions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.12893","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_lakequest_19d4b53f","familyId":"bmf_d4717a70a8cc","name":"LakeQuest","oneLine":"LakeQuest evaluates end-to-end question answering over data lakes with 9,846 QA pairs across three domains (AI/ML metadata, retail banking, biomedical drug info), with exact modality-aware evidence pointers, measuring retrieval and cross-modal synthesis.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences","Finance & Economics"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech","Financial Services"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.12310","pdf":"https://arxiv.org/pdf/2607.12310","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.12310"},"evidence":{"snippet":"To bridge this gap, we introduce LakeQuest, a human-validated benchmark of 9,846 QA pairs designed to evaluate the end-to-end retrieve-and-synthesize pipeline over realistic data lakes.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.12310"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LakeQuest evaluates end-to-end question answering over data lakes with 9,846 QA pairs across three domains (AI/ML metadata, retail banking, biomedical drug info), with exact modality-aware evidence pointers, measuring retrieval and cross-modal synthesis.","whyItMatters":"This benchmark fills the gap in evaluating QA systems on heterogeneous, weakly structured data lakes, exposing failure modes where high-quality retrieval does not guarantee correct reasoning, which is crucial for agentic QA development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fb583d1c3855871c5ebdd7aef9d3540e49cecbff97fa81d849b6ae3193b0be9e"},"motivation":"While modern question answering (QA) systems excel on clean, schema-aligned corpora, real-world knowledge is rarely so neatly packaged.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"Conference on Language Modeling (COLM) 2026","evidence":"24 pages, 4 figures, 18 tables. Accepted at the Conference on Language Modeling (COLM) 2026","evidenceUrl":"https://arxiv.org/abs/2607.12310","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"Conference on Language Modeling (COLM) 2026","reviewStatus":"accepted","decisionRaw":"24 pages, 4 figures, 18 tables. Accepted at the Conference on Language Modeling (COLM) 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.12310","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"24 pages, 4 figures, 18 tables. Accepted at the Conference on Language Modeling (COLM) 2026","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"cross-domain"},{"id":"bm_jitoma-bench_daae8913","familyId":"bmf_ddb468992b92","name":"JITOMA-Bench","oneLine":"JITOMA-Bench is a suite for long-horizon multi-tasking and multi-step reasoning in robotics, focusing on just-in-time scene graph growth to combat perceptual saturation.","area":"Vision & 3D","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.13245","pdf":"https://arxiv.org/pdf/2607.13245","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.13245"},"evidence":{"snippet":"To evaluate these dynamic capabilities and study perceptual saturation trade-offs, we introduce JITOMA-Bench, a comprehensive suite for long-horizon multi-tasking and complex multi-step reasoning.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.13245"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"JITOMA-Bench is a suite for long-horizon multi-tasking and multi-step reasoning in robotics, focusing on just-in-time scene graph growth to combat perceptual saturation.","whyItMatters":"The benchmark supports the JITOMA framework's evaluation, but its primary purpose is to demonstrate the framework's advantages rather than serve as a standalone comparison benchmark.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d1ba2493643e2e1b62f8b60a9ca016d40196b8418369b28a7d5e85a0d0fc872b"},"motivation":"While 3D Scene Graphs (3DSGs) provide crucial structured representations for embodied agents, conventional Ahead-of-Time, build-everything-then-filter pipelines conflict with the real-time, low-latency demands of edge platforms, inducing a perceptual saturation effect via severe observation redundancy.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.13245","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_hedgehog_997bad40","familyId":"bmf_064a43832b4f","name":"HEDGEHOG","oneLine":"HEDGEHOG is a six-stage filtration benchmark for evaluating molecular generators on medicinal plausibility, including physicochemical, structural, synthesis, docking, and 3D pose checks, applied to 23 generators and 230,000 molecules.","area":"Language & Knowledge","applicationDomains":["Science & Research","Health & Life Sciences"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals","Pharma & Biotech"],"capabilities":[],"topics":["cs.LG"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.13155","pdf":"https://arxiv.org/pdf/2607.13155","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.13155"},"evidence":{"snippet":"We introduce HEDGEHOG, a unified six-stage filtration benchmark that is inspired by industrial hit identification workflows: (i) preprocessing; (ii) physicochemical descriptor screening; (iii) structural alerts and graph-sanity checks; (iv) synthesis feasibility; (v) docking and binding affinity estimation; and (vi) three-dimensional pose and interaction checks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.13155"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HEDGEHOG is a six-stage filtration benchmark for evaluating molecular generators on medicinal plausibility, including physicochemical, structural, synthesis, docking, and 3D pose checks, applied to 23 generators and 230,000 molecules.","whyItMatters":"This benchmark exposes that current molecular generators fail to produce compounds that pass all medicinal, synthesis, docking, and 3D filters simultaneously, providing a more realistic evaluation for drug discovery.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ab4bed6f32fa784628a36408bc6ab0132c84c7ce1f1c60b6b1f2075ce176dbcb"},"motivation":"Generative molecular models can support early drug discovery by proposing new candidate compounds de novo.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.13155","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"cross-domain"},{"id":"bm_deep4ge_710e720a","familyId":"bmf_231023f65b39","name":"Deep4ge","oneLine":"Deep4ge is a dataset of 14,227 training runs from 59 DNN programs with documented faults, providing per-epoch features for fault detection and diagnosis tasks, including binary detection, multi-class diagnosis, and early prediction.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.12868","pdf":"https://arxiv.org/pdf/2607.12868","project":"https://doi.org/10.5281/zenodo.20337241","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.12868"},"evidence":{"snippet":"We present Deep4ge, a controlled benchmark of 14,227 training runs generated from 59 adapted TensorFlow/Keras deep neural network (DNN) programs collected from Stack Overflow.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.12868"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Deep4ge is a dataset of 14,227 training runs from 59 DNN programs with documented faults, providing per-epoch features for fault detection and diagnosis tasks, including binary detection, multi-class diagnosis, and early prediction.","whyItMatters":"This fills the gap of a public dataset for diagnosing DNN implementation faults, supporting reproducible research and tool development for fault detection in deep learning systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b906aff384e402d000e7bfc7b4a236bc02bf36a9c4498a943248d7dbc4a649c2"},"motivation":"Deep learning systems often fail due to subtle implementation faults that alter training behavior.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICSME Data & Tool Track, 2026","evidence":"Accepted at ICSME Data & Tool Track, 2026","evidenceUrl":"https://arxiv.org/abs/2607.12868","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICSME Data & Tool Track, 2026","reviewStatus":"accepted","decisionRaw":"Accepted at ICSME Data & Tool Track, 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.12868","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at ICSME Data & Tool Track, 2026","level":"author-claim"}]}],"publishers":[{"name":"Zenodo","organizationType":"community","sourceUrl":"https://doi.org/10.5281/zenodo.20337241","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_sportmv-bench_a6dbc37e","familyId":"bmf_90df5079f183","name":"SportMV-Bench","oneLine":"SportMV-Bench evaluates multi-view sports video understanding with 1022 multi-view bundles and 3015 QA pairs across 10 sports, covering perception, rule-aware event interpretation, and adjudicative reasoning.","area":"Vision & 3D","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.11844","pdf":"https://arxiv.org/pdf/2607.11844","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.11844"},"evidence":{"snippet":"To address this gap, we introduce SportMV-Bench, a comprehensive benchmark built from official match recordings, through a dedicated pipeline combining LLM-based generation, MLLM-based verification, and human filtering to ensure quality and consistency.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.11844"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SportMV-Bench evaluates multi-view sports video understanding with 1022 multi-view bundles and 3015 QA pairs across 10 sports, covering perception, rule-aware event interpretation, and adjudicative reasoning.","whyItMatters":"This benchmark addresses the lack of multi-view sports video evaluation, revealing that MLLMs struggle with fine-grained visual perception and view selection, guiding future development in video reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e662c385e93034586c119dd77ed3b05e762da52087f17d1425c4611aa6c1da22"},"motivation":"Recent Multimodal Large Language Models (MLLMs) achieve strong performance on single-view video understanding benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.11844","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_slvmbench_4b061162","familyId":"bmf_6aaeb103fa63","name":"SLVMBench","oneLine":"SLVMBench evaluates video-LLMs' ability to learn skills from long video memory and apply them to real-time tasks, using 2-3 hour streams with embedded tutorials and human-annotated questions.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.11312","pdf":"https://arxiv.org/pdf/2607.11312","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.11312"},"evidence":{"snippet":"We introduce Skill Learning from Video Memory (SLVMBench), the first benchmark that jointly evaluates whether video large language models (video-LLMs) can learn skills from long video memory and apply them to real-time tasks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.11312"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SLVMBench evaluates video-LLMs' ability to learn skills from long video memory and apply them to real-time tasks, using 2-3 hour streams with embedded tutorials and human-annotated questions.","whyItMatters":"This is the first benchmark to test skill learning from long-context video memory, revealing significant limitations in current video LLMs and providing a realistic evaluation for skill acquisition.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"89196b25fda0c9dce27ac8b3c81878701042f5293681496027a0aeb1a311ba44"},"motivation":"We introduce Skill Learning from Video Memory (SLVMBench), the first benchmark that jointly evaluates whether video large language models (video-LLMs) can learn skills from long video memory and apply them to real-time tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.11312","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_sdabench_9285d4fe","familyId":"bmf_4b6f310c64ce","name":"SDABench","oneLine":"SDABench evaluates LLMs' scientific data analysis capabilities across six capabilities (descriptive, exploratory, inferential, predictive, causal, mechanistic) and five domains, with 527 real and 6000 synthetic instances in multiple-choice and open-ended formats.","area":"Language & Knowledge","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals"],"capabilities":[],"topics":["AI Scientist"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.11079","pdf":"https://arxiv.org/pdf/2607.11079","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.11079"},"evidence":{"snippet":"We introduce SDABench, a benchmark that reorganizes evaluation around six capabilities (descriptive, exploratory, inferential, predictive, causal, and mechanistic) across five domains (Biology, Chemistry, Environment, Geography, Physics).","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":10,"hfDailySubmittedAt":"2026-07-15T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.11079"},"ranking":{"90d":{"score":54,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SDABench evaluates LLMs' scientific data analysis capabilities across six capabilities (descriptive, exploratory, inferential, predictive, causal, mechanistic) and five domains, with 527 real and 6000 synthetic instances in multiple-choice and open-ended formats.","whyItMatters":"This benchmark reveals that LLMs degrade sharply on tasks requiring assumption selection, latent-process modeling, and mechanistic reasoning, highlighting gaps for scientific discovery applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"07fdbbabecc9cf01fcdb9fc894ad3d1ad4a37be1b3a46ad553feef967c6d090b"},"motivation":"Existing benchmarks for scientific data analysis evaluate LLMs primarily on code execution or workflow completion, overlooking that scientific analysis serves to support distinct types of scientific claims: hypothesis exploration, statistical inference, mechanistic explanation, each with different assumptions and validity criteria.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.11079","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_rrs-10k_71143844","familyId":"bmf_76690795f4ea","name":"RRS-10K","oneLine":"RRS-10K is a benchmark for rare remote sensing image interpretation containing 10,738 military-related images with multiple format question-answer pairs, organized into three capability dimensions and 20 leaf tasks covering perception, reasoning, and robustness.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.24810","pdf":"https://arxiv.org/pdf/2607.24810","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.24810"},"evidence":{"snippet":"To address this gap, we present RRS-10K, a benchmark for rare remote sensing image interpretation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.24810"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RRS-10K is a benchmark for rare remote sensing image interpretation containing 10,738 military-related images with multiple format question-answer pairs, organized into three capability dimensions and 20 leaf tasks covering perception, reasoning, and robustness.","whyItMatters":"Current remote sensing benchmarks are dominated by common scenes, limiting understanding of VLM performance on rare, long-tail scenarios. RRS-10K provides a standardized evaluation for this gap, enabling systematic analysis of failure modes and guiding development of more reliable remote sensing VLMs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9a49ac6bb876401f97e9665c5c08748ea21e5812646d387fed08ec59b23ec2b4"},"motivation":"Vision-language models (VLMs) have achieved strong performance on general remote sensing tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.24810","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_researchqa_6b10c7bd","familyId":"bmf_27dd5af619ba","name":"ResearchQA","oneLine":"ResearchQA is a benchmark of 6,211 single-paper question-answer pairs from 494 open-access papers across eight domains, designed for citation-grounded evaluation with multiple valid supporting passages and grounded refusal.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.11074","pdf":"https://arxiv.org/pdf/2607.11074","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.11074"},"evidence":{"snippet":"We introduce ResearchQA, a benchmark of 6,211 single-paper question-answer pairs from 494 open-access papers spanning eight domains and four question types: lookup, comprehension, multi-hop, and adversarial.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.11074"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ResearchQA is a benchmark of 6,211 single-paper question-answer pairs from 494 open-access papers across eight domains, designed for citation-grounded evaluation with multiple valid supporting passages and grounded refusal.","whyItMatters":"Existing evaluation methods often fail to detect whether answers are supported by verifiable citations. ResearchQA provides a standardized way to assess citation accuracy and groundedness, separating systems more clearly than LLM-evaluator scores.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"294ca04530c165a5a5deb556ef6549c260be9dc19623174031b01b9c05ba1bd9"},"motivation":"Large language models are increasingly used to assist scientific reading, but existing evaluation methods often fail to detect whether answers are supported by verifiable citations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.11074","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_needl-bench_c98901f1","familyId":"bmf_825f0496303e","name":"NEEDL-Bench","oneLine":"NEEDL-Bench is a microscopy detection benchmark for Swiss Needle Cast and stomata detection, with 3250 annotated images from 1082 Douglas-fir needles, annotated for keypoint and bounding-box detectors, and two evaluation splits.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.12076","pdf":"https://arxiv.org/pdf/2607.12076","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.12076"},"evidence":{"snippet":"We present NEEDL-Bench, a microscopy detection benchmark for Swiss Needle Cast (SNC), a fungal disease of Douglas-fir trees.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.12076"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"NEEDL-Bench is a microscopy detection benchmark for Swiss Needle Cast and stomata detection, with 3250 annotated images from 1082 Douglas-fir needles, annotated for keypoint and bounding-box detectors, and two evaluation splits.","whyItMatters":"There is no existing dataset for automatic detection of these structures, despite the importance of Douglas-fir. This benchmark provides a standardized evaluation for keypoint and object detection methods, highlighting challenges like small objects and occlusion.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"61ba737d930d2e4dd0e4c14423206d07714785c361a5776a829267275e8238d4"},"motivation":"We present NEEDL-Bench, a microscopy detection benchmark for Swiss Needle Cast (SNC), a fungal disease of Douglas-fir trees.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.12076","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ifcmemorybench_f2c4e9c3","familyId":"bmf_c1b4184d38fa","name":"IFCMemoryBench","oneLine":"IFCMemoryBench evaluates long-term memory in LLM-based agents for BIM information retrieval, with 143 multi-session tasks across 19 projects and 4,016 prior sessions, requiring integration of remembered context with live IFC queries.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.IR"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.26072","pdf":"https://arxiv.org/pdf/2607.26072","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.26072"},"evidence":{"snippet":"We introduce IFCMemoryBench, a benchmark for evaluating long-term memory in LLM-based BIM information retrieval.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.26072"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"IFCMemoryBench evaluates long-term memory in LLM-based agents for BIM information retrieval, with 143 multi-session tasks across 19 projects and 4,016 prior sessions, requiring integration of remembered context with live IFC queries.","whyItMatters":"Existing memory evaluations focus on conversational recall, not professional domains. IFCMemoryBench provides a test for whether agents can reuse information across sessions in a structured, domain-specific environment, revealing gaps in current memory systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"130a6741595f56737952fc8d1620137b90d8b59ba67b1586753db1347b86e2a4"},"motivation":"Long-term memory is becoming a core capability of LLM-based agents, but existing evaluations largely test conversational recall in open-domain or persona-grounded settings.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.26072","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_deepbias_70d900bc","familyId":"bmf_dc5ea037d610","name":"DeepBias","oneLine":"DeepBiasBench is a benchmark for in-depth probing of social biases in LVLMs, built using an adaptive framework with dynamic test data generation and iterative rewriting.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CY"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.11228","pdf":"https://arxiv.org/pdf/2607.11228","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.11228"},"evidence":{"snippet":"Furthermore, we build a benchmark named DeepBiasBench using our framework.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.11228"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DeepBiasBench is a benchmark for in-depth probing of social biases in LVLMs, built using an adaptive framework with dynamic test data generation and iterative rewriting.","whyItMatters":"Static bias datasets provide only superficial assessment. DeepBiasBench aims to expose deeper biases through adaptive probing, but its dynamic nature and lack of fixed protocol make it unsuitable for standalone comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a21f1364d73bc7540a039ebc773cc0bd94224880f5fa2a47782329bf551e4f45"},"motivation":"While Large Vision-Language Models (LVLMs) demonstrate remarkable capabilities, they remain highly susceptible to embedded social biases.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.11228","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_backendforge_1bb8c499","familyId":"bmf_6ecda39e082a","name":"BackendForge","oneLine":"BackendForge is a benchmark of 56 contract-defined backend generation tasks from real open-source applications. LLMs must generate Dockerized services evaluated through HTTP tests against an OpenAPI contract.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation"],"topics":["Code"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.11042","pdf":"https://arxiv.org/pdf/2607.11042","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.11042"},"evidence":{"snippet":"We introduce BackendForge, a benchmark of 56 contract-defined backend generation tasks rewritten from real open-source applications.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.11042"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"BackendForge is a benchmark of 56 contract-defined backend generation tasks from real open-source applications. LLMs must generate Dockerized services evaluated through HTTP tests against an OpenAPI contract.","whyItMatters":"Agentic LLMs need to produce deployable and behaviorally correct software artifacts. BackendForge provides a deterministic, black-box evaluation of backend service generation, exposing gaps between local API implementation and complete service delivery.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"740147521fd047ec3109764fbe7e21679177245b00575ec9f9939387fa9b9ada"},"motivation":"Large language models (LLMs) are increasingly used in agentic coding settings, where they can inspect files, execute commands, run tests, observe failures, and iteratively revise code.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.11042","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_advancedmathbench_22d95486","familyId":"bmf_aa5a0a0d601b","name":"AdvancedMathBench","oneLine":"AdvancedMathBench is a benchmark suite for advanced mathematical reasoning, containing ProverBench (296 proof problems) and VerifierBench (888 proof trajectories with expert labels), with an automatic verification pipeline.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.11849","pdf":"https://arxiv.org/pdf/2607.11849","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.11849"},"evidence":{"snippet":"To bridge this gap, we introduce AdvancedMathBench, a benchmark suite designed to evaluate advanced mathematical reasoning capabilities.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":33,"hfDailySubmittedAt":"2026-07-14T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.11849"},"ranking":{"90d":{"score":56,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"AdvancedMathBench is a benchmark suite for advanced mathematical reasoning, containing ProverBench (296 proof problems) and VerifierBench (888 proof trajectories with expert labels), with an automatic verification pipeline.","whyItMatters":"Existing math benchmarks focus on high-school levels and final answers. AdvancedMathBench evaluates proof generation and verification at advanced levels, providing fine-grained assessments of proof correctness and error detection.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cf8c4038b5cb13c25589b51cafb5222643936f34d4177439d46d3b8dfc07f355"},"motivation":"Large language models (LLMs) have achieved remarkable performance on high-school and olympiad-style mathematics, yet their capabilities on advanced mathematics remain poorly understood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.11849","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_opti-agent-bench_f017ef81","familyId":"bmf_4e5dce3ea76c","name":"Opti-Agent-Bench","oneLine":"Opti-Agent-Bench evaluates LLM agents on the end-to-end optimization R&D pipeline, from business-language description to mathematical modeling, algorithm selection, code implementation, and report generation, with modular evaluation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.10768","pdf":"https://arxiv.org/pdf/2607.10768","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.10768"},"evidence":{"snippet":"We introduce Opti-Agent-Bench, an end-to-end benchmark that evaluates Large Language Models (LLMs) across the complete optimization R&D pipeline, from understanding business-language descriptions through mathematical modeling, algorithm selection, and code implementation, to solution report generation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.10768"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Opti-Agent-Bench evaluates LLM agents on the end-to-end optimization R&D pipeline, from business-language description to mathematical modeling, algorithm selection, code implementation, and report generation, with modular evaluation.","whyItMatters":"Optimization benchmarks typically test pre-structured formulations. Opti-Agent-Bench assesses the full pipeline, exposing failure modes like constraint omission and model-code inconsistency invisible under single-metric evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ac89ccae89caa72d72bef818a582d6b749680d08e1bbb7869dc78f93157d3bc9"},"motivation":"LLM-based agents are increasingly deployed to solve optimization problems, yet existing benchmarks evaluate them on pre-structured mathematical formulations that bypass the most critical challenge: translating complex business requirements into correct models and solve efficiently.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.10768","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_mafiascope_6f84f9bf","familyId":"bmf_a84e60ac04f2","name":"MafiaScope","oneLine":"MafiaScope evaluates LLM agents in the social deduction game Mafia. It probes each agent's private beliefs after every public utterance, scoring them against ground truth without influencing the game. The testbed provides an open-source engine, interactive visualizer, recorded games, and counterfactual replay corpus.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.10645","pdf":"https://arxiv.org/pdf/2607.10645","project":"https://karpovilia.github.io/mafiascope/","code":"https://github.com/karpovilia/mafiascope","data":null,"hfPaper":"https://huggingface.co/papers/2607.10645"},"evidence":{"snippet":"We present MafiaScope, an open testbed that turns the social deduction game Mafia into a measurement instrument for machine Theory of Mind.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.10645"},"ranking":{"90d":{"score":29,"rank":367,"coverage":0.55,"confidence":"Low"}},"description":"MafiaScope evaluates LLM agents in the social deduction game Mafia. It probes each agent's private beliefs after every public utterance, scoring them against ground truth without influencing the game. The testbed provides an open-source engine, interactive visualizer, recorded games, and counterfactual replay corpus.","whyItMatters":"Machine Theory of Mind is difficult to measure from observable behavior alone. MafiaScope separates incorrect belief formation from incorrect action under correct beliefs, a distinction invisible in dialogue or outcome data, enabling more precise evaluation of social reasoning in LLM agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e824aa18d8728d8ce133e2b350308995e6708abafda4ae2fe48e5e64a53b88ea"},"motivation":"An LLM agent's public behaviour reveals little about its social reasoning: an agent that votes correctly may be guessing, and an agent that lies well leaves no trace of what it actually believes.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.10645","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MafiaScope Team","organizationType":"academic-lab","sourceUrl":"https://github.com/karpovilia/mafiascope","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_imaging-101_1fefb66e","familyId":"bmf_1df9e5402412","name":"Imaging-101","oneLine":"Imaging-101 evaluates coding agents on 57 computational imaging tasks across six scientific domains, with three tracks for planning, function-level unit tests, and end-to-end reconstruction.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.10789","pdf":"https://arxiv.org/pdf/2607.10789","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.10789"},"evidence":{"snippet":"We introduce Imaging-101, a benchmark of 57 expert-verified computational imaging tasks spanning six scientific domains, each grounded in a peer-reviewed paper and canonicalized into a standardized four-stage pipeline (preprocessing, forward physics modeling, inverse solver, and visualization) Three evaluation tracks (planning, function-level unit tests, and end-to-end reconstruction) probe distinct agent capabilities across the full pipeline.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.10789"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Imaging-101 evaluates coding agents on 57 computational imaging tasks across six scientific domains, with three tracks for planning, function-level unit tests, and end-to-end reconstruction.","whyItMatters":"General coding benchmarks may not capture domain-specific challenges in scientific imaging, and this benchmark could help assess agent capabilities in algorithm selection, physical conventions, and pipeline integration.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c883469cf9d0a86629947ee0c300addad297d829ad6d3654eb9cffd8598205d6"},"motivation":"Computational imaging, which recovers hidden signals from indirect, noisy measurements, underpins quantitative discovery across scientific disciplines, yet building a correct reconstruction pipeline demands deep domain expertise and remains laborious even for domain scientists.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.10789","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_vvm-bench_539536ab","familyId":"bmf_639ec828a5aa","name":"VVM-Bench","oneLine":"VVM-Bench evaluates Large Multimodal Models on semantic perception and modality understanding across six real and synthetic modalities, using multiple-choice questions and generation tasks to assess zero-shot generalization to unseen visual modalities.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-07-11","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.10308","pdf":"https://arxiv.org/pdf/2607.10308","project":null,"code":"https://github.com/Hunter-Will/VVM-Tuning","data":null,"hfPaper":"https://huggingface.co/papers/2607.10308"},"evidence":{"snippet":"To facilitate research in this direction, we introduce VVM-Bench, a comprehensive benchmark featuring 6 real and synthetic modalities to evaluate semantic perception and modality understanding.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.10308"},"ranking":{"90d":{"score":25,"rank":388,"coverage":0.7,"confidence":"Medium"}},"description":"VVM-Bench evaluates Large Multimodal Models on semantic perception and modality understanding across six real and synthetic modalities, using multiple-choice questions and generation tasks to assess zero-shot generalization to unseen visual modalities.","whyItMatters":"VVM-Bench provides a standardized protocol for assessing LMMs' ability to generalize across visual modalities, enabling comparison of current models on a crucial capability for real-world deployment where sensor types vary.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c76694826111f1cfc36a1e719f1ef9a93dc423d53d11b6557795d97a790a8f7d"},"motivation":"Despite the advancements of Large Multimodal Models (LMMs) in RGB vision, their ability to generalize to unseen visual modalities remains a largely unexplored challenge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"European Conference on Computer Vision (ECCV) 2026","evidence":"Accepted by the European Conference on Computer Vision (ECCV) 2026","evidenceUrl":"https://arxiv.org/abs/2607.10308","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"European Conference on Computer Vision (ECCV) 2026","reviewStatus":"accepted","decisionRaw":"Accepted by the European Conference on Computer Vision (ECCV) 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.10308","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by the European Conference on Computer Vision (ECCV) 2026","level":"author-claim"}]}],"publishers":[{"name":"VVM-Tuning project","organizationType":"academic-lab","sourceUrl":"https://github.com/Hunter-Will/VVM-Tuning","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_synthdocbench_7af1a05d","familyId":"bmf_3a3c4b0877bd","name":"SynthDocBench","oneLine":"SynthDocBench evaluates vision-language models on 1,788 questions over 200 synthetic long-context documents (avg. 51.1 pages) with 3,340 charts. It varies document length, layout archetype, modality composition, and question type as independent controlled factors, across chart, cross-modal, and complex subsets with deterministic ground truth.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Long Context"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-07-11","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.10400","pdf":"https://arxiv.org/pdf/2607.10400","project":null,"code":"https://github.com/ServiceNow/SynthDocBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.10400"},"evidence":{"snippet":"We introduce SynthDocBench, a fully synthetic benchmark for long-context visual document understanding that systematically controls factors including document length, layout structure, modality composition, and question type.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":71,"hfDailySubmittedAt":"2026-07-15T00:00:00.000Z","githubStars":9,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.10400"},"ranking":{"90d":{"score":64,"rank":105,"coverage":0.7,"confidence":"Medium"}},"description":"SynthDocBench evaluates vision-language models on 1,788 questions over 200 synthetic long-context documents (avg. 51.1 pages) with 3,340 charts. It varies document length, layout archetype, modality composition, and question type as independent controlled factors, across chart, cross-modal, and complex subsets with deterministic ground truth.","whyItMatters":"Existing document benchmarks confound length, layout, and modality, obscuring specific model failure causes. SynthDocBench provides controlled attribution of failures, revealing sharp degradation with document length, positional sensitivity in the middle third, and breakdown of chart comprehension in long documents, which is valuable for targeted model improvement.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a77aa761d725adc596250e12d35b6b03c1d0b02ed127e66a50dd9ccf9c1383c2"},"motivation":"Vision language models (VLMs) have achieved strong performance on visual document understanding benchmarks such as DocVQA, ChartQA, and MMLongBench-Doc.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"COLM 2026","evidence":"29 Pages, 27 Tables, 13 Figures, Accepted at COLM 2026","evidenceUrl":"https://arxiv.org/abs/2607.10400","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"COLM 2026","reviewStatus":"accepted","decisionRaw":"29 Pages, 27 Tables, 13 Figures, Accepted at COLM 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.10400","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"29 Pages, 27 Tables, 13 Figures, Accepted at COLM 2026","level":"author-claim"}]}],"publishers":[{"name":"ServiceNow AI","organizationType":"company-research-lab","sourceUrl":"https://github.com/ServiceNow/SynthDocBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception","Long Context & Memory"],"domainScope":"general"},{"id":"bm_netinjectbench_1ee2f598","familyId":"bmf_e8dac394667e","name":"NetInjectBench","oneLine":"NetInjectBench evaluates LLM agents for network operations under indirect prompt injection. It comprises 130 scenarios: 40 benign, 40 weak-attack, 40 strong-attack, and 10 approved high-impact changes, with separated untrusted artifact text, trusted policy metadata, and evaluation labels. Scoring measures unsafe tool-action rate, usefulness, and overblocking across models and defenses.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Tool use"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.10490","pdf":"https://arxiv.org/pdf/2607.10490","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.10490"},"evidence":{"snippet":"We present NetInjectBench, a 130-scenario benchmark that separates untrusted artifact text, trusted policy metadata, and evaluation labels for network-operation tool use.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.10490"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"NetInjectBench evaluates LLM agents for network operations under indirect prompt injection. It comprises 130 scenarios: 40 benign, 40 weak-attack, 40 strong-attack, and 10 approved high-impact changes, with separated untrusted artifact text, trusted policy metadata, and evaluation labels. Scoring measures unsafe tool-action rate, usefulness, and overblocking across models and defenses.","whyItMatters":"Network operations increasingly rely on tool-using LLM agents, which are vulnerable to indirect prompt injections via untrusted artifacts. This benchmark quantifies safety and usefulness trade-offs in a realistic network-domain environment, providing a concrete means to assess defense effectiveness and authorization boundaries.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3485d8e24f03ea71ead158ee6e58d3d801744dc00caccf50b513fe55e82b8ace"},"motivation":"Tool-using large language model (LLM) agents are attractive for network operations, but tickets, alerts, logs, runbooks, and ChatOps messages can carry indirect prompt injections.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.10490","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents","Tool Calling"],"domainScope":"general"},{"id":"bm_mag_3c2b6829","familyId":"bmf_f8685538bda2","name":"MAG","oneLine":"MAG is a benchmark that unifies task execution and guide writing into a single multimodal action and guide task, with grounding over screenshots. It includes a harness for annotation, training, evaluation, and joint metrics.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents","Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.10079","pdf":"https://arxiv.org/pdf/2607.10079","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.10079"},"evidence":{"snippet":"In this work we introduce MAG, the first benchmark that unifies task execution and guide writing into a single Multimodal Action and Guide task, with two grounding schemes over screenshots: Set-of-Mark element selection and raw pixel coordinates.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.10079"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MAG is a benchmark that unifies task execution and guide writing into a single multimodal action and guide task, with grounding over screenshots. It includes a harness for annotation, training, evaluation, and joint metrics.","whyItMatters":"The evaluation gap is that prior benchmarks separate web agent actions and guide text generation, and often rely on textual DOM rather than screenshots. MAG provides a unified evaluation for multimodal understanding and generation in live environments, which could support development of more capable web agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f07973bc7589c7ceb062e24adbecc848593c2bb8b8b2d41f9e3336833c1996ce"},"motivation":"Digital Adoption Platforms (DAPs) are embedded overlays widely used on web systems to guide users through operations inside a page, helping them get started with unfamiliar interfaces quickly.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.10079","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_force-bench_be1ccc66","familyId":"bmf_3be86d47de12","name":"FORCE-Bench","oneLine":"FORCE-Bench evaluates agentic AI systems in enterprise finance across three task types: financial obligation research, financial entity performance research, and business brief generation. It includes 251 expert-annotated queries and a rubric-based scoring framework across eight dimensions: accuracy, citations, clarity, depth, groundedness, recency, relevance, and structure.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19409","pdf":"https://arxiv.org/pdf/2607.19409","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19409"},"evidence":{"snippet":"We introduce FORCE-Bench, which contains 251 expert-annotated queries and evaluates responses using a rubric-based framework calibrated to the requirements of the operational finance domain, across eight dimensions: accuracy, citations, clarity, depth, groundedness, recency, relevance, and structure.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19409"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FORCE-Bench evaluates agentic AI systems in enterprise finance across three task types: financial obligation research, financial entity performance research, and business brief generation. It includes 251 expert-annotated queries and a rubric-based scoring framework across eight dimensions: accuracy, citations, clarity, depth, groundedness, recency, relevance, and structure.","whyItMatters":"Existing benchmarks focus on general capabilities rather than operational finance workflows. FORCE-Bench provides a domain-specific evaluation tool that measures rule adherence, verifiability, and groundedness, which are critical for real-world deployment of agentic systems in finance. It enables comparable assessment of agent performance under operational constraints.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"23431856bd791d1004934d5b3e7a259a952219e0989a2d3e6faf7295842eb1a5"},"motivation":"Recent advances in large language models have accelerated deployment of agentic systems in operational finance.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19409","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FORCE-Bench team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.19409","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_eyt-bench_32cb0b9c","familyId":"bmf_67a637fc68e9","name":"EYT-Bench","oneLine":"EYT-Bench evaluates multi-turn dialogue capabilities of LLMs using a decoupled three-party setup with user simulation, target modeling, and judging.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.10428","pdf":"https://arxiv.org/pdf/2607.10428","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.10428"},"evidence":{"snippet":"We introduce EYT-Bench, a human-centered benchmark whose evaluation protocol is built around a decoupled three-party design: a persona-grounded user simulator, a target model evaluated on both intent perception and response generation, and an independent, configurable ensemble of LLM judges.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.10428"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"EYT-Bench evaluates multi-turn dialogue capabilities of LLMs using a decoupled three-party setup with user simulation, target modeling, and judging.","whyItMatters":"Assessing conversational AI beyond single turns may guide development of more consistent and context-aware dialogue systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"10404f1d93198eb060bae2c1f9faa113cc8879b19b7cc5375fd9b11ece7f035a"},"motivation":"Evaluating large language models (LLMs) as multi-turn conversational partners requires probing capabilities that single-turn benchmarks miss: persona consistency, evolving intent tracking, emotional dynamics, and goal completion across many turns.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.10428","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_chartsync_6261046a","familyId":"bmf_700e28d4e4ca","name":"ChartSync","oneLine":"ChartSync evaluates image editing models on chart editing tasks, including text-only edits and visuo-logical cascading edits (VLCE) requiring synchronized text and geometry changes. The benchmark contains 870 expert-validated triplets across nine chart families, with objective visual metrics and a vision-language model judge for evaluation.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.10301","pdf":"https://arxiv.org/pdf/2607.10301","project":null,"code":"https://github.com/kaka-yjk/ChartSyncCodebase","data":null,"hfPaper":"https://huggingface.co/papers/2607.10301"},"evidence":{"snippet":"To systematically evaluate this capability, we introduce ChartSync, an expert-validated benchmark constructed via a programmatic rendering pipeline that guarantees deterministic visuo-logical coupling for the ground truth.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.10301"},"ranking":{"90d":{"score":49,"rank":199,"coverage":0.7,"confidence":"Medium"}},"description":"ChartSync evaluates image editing models on chart editing tasks, including text-only edits and visuo-logical cascading edits (VLCE) requiring synchronized text and geometry changes. The benchmark contains 870 expert-validated triplets across nine chart families, with objective visual metrics and a vision-language model judge for evaluation.","whyItMatters":"Existing image editing benchmarks often overlook structured data charts where data modifications require geometric synchronization. ChartSync provides a reproducible evaluation to assess models' ability to handle dependency-aware cascading updates, revealing capability gaps in open-source versus proprietary models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"353834346449c1c354ef5316925b669a576cb6aeb0cd776ee64d32ca80034f5b"},"motivation":"Generative image editing models struggle with structured statistical charts when data modifications require geometric synchronization.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.10301","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ChartSync Team","organizationType":"academic-lab","sourceUrl":"https://github.com/kaka-yjk/ChartSyncCodebase","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_agentabstain_d94f5610","familyId":"bmf_4b74dce329f5","name":"AgentAbstain","oneLine":"AgentAbstain is a paired-task benchmark for evaluating LLM agents' ability to abstain from acting in scenarios such as ambiguity, conflicting constraints, or tool failures. It includes 263 paired tasks across 42 sandbox environments, with a proposed pipeline for generating fresh task instances.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Agents","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-11","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.10059","pdf":"https://arxiv.org/pdf/2607.10059","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.10059"},"evidence":{"snippet":"At its core, AgentAbstain is a paired-task benchmark built on an agent-native taxonomy of 8 abstention scenarios across pre-execution reasoning and runtime discovery.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.10059"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"AgentAbstain is a paired-task benchmark for evaluating LLM agents' ability to abstain from acting in scenarios such as ambiguity, conflicting constraints, or tool failures. It includes 263 paired tasks across 42 sandbox environments, with a proposed pipeline for generating fresh task instances.","whyItMatters":"Agent abstention is critical for safe deployment, yet existing evaluations focus on task success. This benchmark targets the gap in measuring calibrated abstention, highlighting that abstention capability is independent of general task-solving ability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"72e3cdd0dcd558b7f52a77710bf5c5d8655a5f4569db27a6fc48214e0bed26dd"},"motivation":"Agent systems based on large language models (LLMs) are increasingly deployed for autonomous tasks, yet existing evaluations mostly focus on task success rather than whether agents know when to abstain.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.10059","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_activefly-bench_205c2488","familyId":"bmf_c957b3cef25e","name":"ActiveFly-Bench","oneLine":"ActiveFly-Bench is a benchmark for UAV embodied perception, decomposing active perception into three tasks: Aerial Embodied Question Answering (Air-EQA), Observation Behavior Planning (OBP), and Fine-grained Language-guided UAV Control (FLUC). It includes datasets from real-world and simulated outdoor environments.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.10180","pdf":"https://arxiv.org/pdf/2607.10180","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.10180"},"evidence":{"snippet":"We introduce ActiveFly-Bench, the first benchmark to bridge cyberspace reasoning and physical-world interaction for UAV embodied perception.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.10180"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ActiveFly-Bench is a benchmark for UAV embodied perception, decomposing active perception into three tasks: Aerial Embodied Question Answering (Air-EQA), Observation Behavior Planning (OBP), and Fine-grained Language-guided UAV Control (FLUC). It includes datasets from real-world and simulated outdoor environments.","whyItMatters":"Active perception in UAVs requires bridging high-level reasoning with low-level control, which current VLMs and VLA models struggle with. This benchmark provides a testbed for embodied aerial intelligence, addressing the lack of benchmarks in this domain.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1bff2991bf2edec25de36a25b1cac71779617d5eab15572fb65c357a7605d9df"},"motivation":"We introduce ActiveFly-Bench, the first benchmark to bridge cyberspace reasoning and physical-world interaction for UAV embodied perception.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.10180","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_wildtrace_951518d8","familyId":"bmf_31889bba386b","name":"WildTrace","oneLine":"WildTrace evaluates long-context reasoning over naturally occurring long-form sources, focusing on integrating evidence dispersed across distant passages. It includes 481 tasks over 214 sources with seven source-internal evidence geometries, with multi-stage validation ensuring answer groundedness and clue necessity.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Long Context","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.09328","pdf":"https://arxiv.org/pdf/2607.09328","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.09328"},"evidence":{"snippet":"We introduce WILDTRACE, a benchmark of 481 tasks over 214 naturally occurring long-form sources such as technical incident reports and lesser-known literary narratives, where all evidence trails arise from the document's own causal, temporal, and narrative logic.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.09328"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"WildTrace evaluates long-context reasoning over naturally occurring long-form sources, focusing on integrating evidence dispersed across distant passages. It includes 481 tasks over 214 sources with seven source-internal evidence geometries, with multi-stage validation ensuring answer groundedness and clue necessity.","whyItMatters":"Existing long-context benchmarks rely on synthetic evidence that may not reflect real source-internal integration. WildTrace provides a more naturalistic evaluation of analytical reading, addressing a gap in assessing true source reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"83be741e07b82a5f872a01d13afb68a5999c14faa723c202c069d955a2f4b018"},"motivation":"Answering complex questions over long documents frequently requires integrating evidence that the source itself disperses naturally across distant passages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.09328","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Long Context & Memory"],"domainScope":"general"},{"id":"bm_who-when-pro_d21cef2d","familyId":"bmf_164e8ad33d5b","name":"Who&When Pro","oneLine":"Who&When Pro is a benchmark for automated failure attribution in agentic systems, containing 12,326 failed trajectories with golden labels across 3 modalities and 26 benchmarks. It uses a controlled pipeline that injects failures after replaying successful prefixes.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.09996","pdf":"https://arxiv.org/pdf/2607.09996","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.09996"},"evidence":{"snippet":"We introduce Who&When Pro, a large-scale benchmark for automated failure attribution in agentic systems.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.09996"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Who&When Pro is a benchmark for automated failure attribution in agentic systems, containing 12,326 failed trajectories with golden labels across 3 modalities and 26 benchmarks. It uses a controlled pipeline that injects failures after replaying successful prefixes.","whyItMatters":"As agents become more capable, automated failure attribution is crucial for debugging and safety. This benchmark provides a large-scale evaluation to guide the development of systems that can identify where and why failures occur.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"adbcb813bc5a8b17661221c59657a23bb9bfcf7b3d09e35853ba77e206e65ab8"},"motivation":"Automated failure attribution uses LLMs to identify where and why agentic systems fail.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.09996","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_tsai-metafraud_9ef6a40f","familyId":"bmf_f0cefb4e8b91","name":"TSAI-MetaFraud","oneLine":"TSAI-MetaFraud is a multimodal, multi-task benchmark dataset for fraud detection in virtual economies, integrating behavioral, transactional, and graph-structured data. It defines tasks including fraud detection, node classification, temporal link prediction, and weakly supervised fraud detection, with baseline evaluations.","area":"Multimodal","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.09528","pdf":"https://arxiv.org/pdf/2607.09528","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.09528"},"evidence":{"snippet":"To address this gap, we present TSAI-MetaFraud, a multimodal, multi-task benchmark dataset for fraud analytics in virtual economies.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.09528"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TSAI-MetaFraud is a multimodal, multi-task benchmark dataset for fraud detection in virtual economies, integrating behavioral, transactional, and graph-structured data. It defines tasks including fraud detection, node classification, temporal link prediction, and weakly supervised fraud detection, with baseline evaluations.","whyItMatters":"Fraud in metaverse ecosystems is a new challenge that combines behavioral and financial data. TSAI-MetaFraud provides a unified benchmark to advance multimodal learning and fraud analytics in virtual economies, filling a gap in existing datasets.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f74e2f7e191243c47e047ab633312cd461986a90916a99261771bff911325246"},"motivation":"The emergence of metaverse platforms has created virtual economies that introduce new challenges related to fraud, bot activity, and illicit financial behavior.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.09528","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_tactidex_18e67e16","familyId":"bmf_e1d0d3e790ce","name":"TactiDex","oneLine":"TactiDex is a real-world tactile-guided benchmark for dexterous manipulation, aligning whole-hand tactile signals with kinematic and object states. It provides standardized evaluation metrics and a framework for tactile-driven transfer, including experiments on single and bimanual tasks.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.09190","pdf":"https://arxiv.org/pdf/2607.09190","project":"https://tactidex.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.09190"},"evidence":{"snippet":"To address this, we introduce TactiDex, a real-world tactile-guided benchmark specifically designed to move dexterous manipulation beyond kinematic mimicry toward contact-level human-likeness.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.09190"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TactiDex is a real-world tactile-guided benchmark for dexterous manipulation, aligning whole-hand tactile signals with kinematic and object states. It provides standardized evaluation metrics and a framework for tactile-driven transfer, including experiments on single and bimanual tasks.","whyItMatters":"Tactile feedback is essential for human-like dexterous manipulation, yet existing benchmarks focus on kinematic imitation. TactiDex addresses this gap by providing a benchmark that emphasizes contact-level human-likeness, enabling physically grounded robot execution.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0c6ae15678ee84d9a6112c5c5634af9feeb5fc7e7f58105b01022e2dfa54a959"},"motivation":"Tactile feedback is fundamental to Hand-Object Interaction (HOI), governing contact formation, force regulation, and stable manipulation, making it essential for achieving true human-like dexterous manipulation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.09190","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TactiDex Team","organizationType":"academic-lab","sourceUrl":"https://tactidex.github.io/","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_prompting-mammalps_0a70a842","familyId":"bmf_1eff7b878620","name":"Prompting-MammAlps","oneLine":"Prompting-MammAlps is a camera-trap text-to-video retrieval benchmark, evaluating video-language models on fine-grained retrieval of ecological events. It includes a test set of 135 queries and 775 candidate videos, with a proposed method that combines action localization with LLM-based parsing.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.09876","pdf":"https://arxiv.org/pdf/2607.09876","project":"https://cnai.epfl.ch/prompting-mammalps","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.09876"},"evidence":{"snippet":"In this work, we introduce Prompting-MammAlps, the first camera-trap TVR benchmark, and propose a fine-grained and interpretable TVR method.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.09876"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Prompting-MammAlps is a camera-trap text-to-video retrieval benchmark, evaluating video-language models on fine-grained retrieval of ecological events. It includes a test set of 135 queries and 775 candidate videos, with a proposed method that combines action localization with LLM-based parsing.","whyItMatters":"Text-to-video retrieval in ecological domains requires spatiotemporal understanding that current VLMs lack. This benchmark provides a standardized evaluation for fine-grained and interpretable retrieval, highlighting the limitations of zero-shot VLMs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b3dfdf6acd5bc7b3d4c219c330c950e703448df60f9ede43f16918f2edc71528"},"motivation":"Automatically retrieving videos from large camera-trap datasets remains challenging.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026","evidence":"Accepted at ECCV 2026; Project page: https://cnai.epfl.ch/prompting-mammalps","evidenceUrl":"https://arxiv.org/abs/2607.09876","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026","reviewStatus":"accepted","decisionRaw":"Accepted at ECCV 2026; Project page: https://cnai.epfl.ch/prompting-mammalps","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.09876","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at ECCV 2026; Project page: https://cnai.epfl.ch/prompting-mammalps","level":"author-claim"}]}],"publishers":[{"name":"EPFL CNAI","organizationType":"academic-lab","sourceUrl":"https://cnai.epfl.ch/prompting-mammalps","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception","Search & Retrieval"],"domainScope":"general"},{"id":"bm_longmedbench_c02feb5e","familyId":"bmf_b4036f59b4ef","name":"LongMedBench","oneLine":"A benchmark for long-horizon clinical decision-making using EHR data from MIMIC-IV, comprising 335 patients with multi-session interactions and three evaluation suites: fact-based QA, temporal reasoning, and long-horizon decision-making.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-07-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.09322","pdf":"https://arxiv.org/pdf/2607.09322","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.09322"},"evidence":{"snippet":"In this work, we introduce LongMedBench, a real-world EHR-based benchmark for long-horizon clinical decision-making.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.09322"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark for long-horizon clinical decision-making using EHR data from MIMIC-IV, comprising 335 patients with multi-session interactions and three evaluation suites: fact-based QA, temporal reasoning, and long-horizon decision-making.","whyItMatters":"Current medical agent evaluations emphasize short-context tasks, while real clinical care requires aggregating evidence over extended periods. This benchmark addresses the need for realistic long-horizon assessment, but the paper does not specify if the benchmark is publicly available for reuse or ongoing submission.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"12cf5e96de684d21fbe481b3e835415b102135ad2f514bdca70765f663539dc9"},"motivation":"In this work, we introduce LongMedBench, a real-world EHR-based benchmark for long-horizon clinical decision-making.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.09322","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_confidencebench_21fa8523","familyId":"bmf_b9ca48dc2fb1","name":"ConfidenceBench","oneLine":"A calibration benchmark evaluating verbalized confidence estimates in frontier LLMs using Brier scores across 200 multiple-choice questions in four categories. Scores are elicited via prompting without logits, applicable to closed and open models.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20526","pdf":"https://arxiv.org/pdf/2607.20526","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20526"},"evidence":{"snippet":"We present ConfidenceBench, a calibration benchmark that evaluates verbalized confidence estimates in 15 frontier LLMs using the Brier score, a proper scoring rule that incentivises truthful probability reporting.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20526"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A calibration benchmark evaluating verbalized confidence estimates in frontier LLMs using Brier scores across 200 multiple-choice questions in four categories. Scores are elicited via prompting without logits, applicable to closed and open models.","whyItMatters":"The benchmark addresses the need to assess model calibration separately from accuracy, which is critical for trustworthy deployment. The private nature of the questions and lack of public artifacts prevent other teams from running or inspecting the benchmark, limiting its standalone utility.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f1f68150dee752cb5c5721bba768cead5af118428559027ee8c9d7762480569b"},"motivation":"Large language models (LLMs) are increasingly deployed in settings where fluent but incorrect answers can be costly.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20526","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_clir-bench_e1c31f75","familyId":"bmf_eab8b70b905c","name":"CLIR-Bench","oneLine":"A benchmark for question answering over irregular clinical time series from ICU records, containing 6,600 QA instances across 11 clinical variables and 11 tasks. It evaluates answer accuracy and evidence use through explicit temporal evidence and answer derivation rules.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.09880","pdf":"https://arxiv.org/pdf/2607.09880","project":null,"code":null,"data":"https://huggingface.co/datasets/winall/CLIR-Bench","hfPaper":"https://huggingface.co/papers/2607.09880"},"evidence":{"snippet":"To fill this gap, we introduce CLIR-Bench, a benchmark for irregular clinical time series QA constructed from de-identified ICU records through a principled four-stage pipeline.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":102,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2607.09880"},"ranking":{"90d":{"score":43,"rank":252,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":31,"datasetRankPopulation":51}},"description":"A benchmark for question answering over irregular clinical time series from ICU records, containing 6,600 QA instances across 11 clinical variables and 11 tasks. It evaluates answer accuracy and evidence use through explicit temporal evidence and answer derivation rules.","whyItMatters":"Existing benchmarks focus on regular time-series or static medical QA, while real ICU data is sparse and asynchronous. This benchmark provides a way to assess whether models can reason over irregular temporal evidence, addressing a gap in clinical NLP evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5dbfca2cc4764cc24ea4cbcdeb5f3380d85699a013877f6ee49c5f0213f3e6af"},"motivation":"Clinical time series are central to patient monitoring, risk assessment, and clinical decision support.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.09880","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"CLIR-Bench Team","organizationType":"academic-lab","sourceUrl":"https://huggingface.co/datasets/winall/CLIR-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_waspmot_953ea5b1","familyId":"bmf_c1b6a961c222","name":"WaspMOT","oneLine":"A benchmark for long-term multi-object tracking of Trichogramma wasps in controlled ecological experiments, with 10 sequences of ~12,000 frames each and dense annotations. It evaluates identity preservation over extended durations.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.08729","pdf":"https://arxiv.org/pdf/2607.08729","project":null,"code":"https://github.com/tstanczyk95/WaspMOT/","data":null,"hfPaper":"https://huggingface.co/papers/2607.08729"},"evidence":{"snippet":"We introduce WaspMOT, a benchmark designed to address this gap through long-duration tracking of Trichogramma wasps in controlled ecological experiments.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08729"},"ranking":{"90d":{"score":29,"rank":368,"coverage":0.55,"confidence":"Low"}},"description":"A benchmark for long-term multi-object tracking of Trichogramma wasps in controlled ecological experiments, with 10 sequences of ~12,000 frames each and dense annotations. It evaluates identity preservation over extended durations.","whyItMatters":"Existing MOT benchmarks focus on short videos, which do not assess long-term identity preservation. This benchmark provides a controlled scenario with closed-set tracking, potentially revealing limitations in current methods that are not observable in conventional datasets.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6741da454f366277ac0714d7dd5188deaace448bb17b8d19baa950289deb744e"},"motivation":"Multi-object tracking (MOT) has achieved strong performance on benchmarks dominated by short video sequences.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"publication_reported","venue":"AVSS 2026","evidence":"AVSS 2026","evidenceUrl":"https://arxiv.org/abs/2607.08729","source":"arxiv-journal-reference","evidenceLevel":"strong-author-metadata","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publications":[{"venueName":"AVSS 2026","publicationStatus":"published","evidence":[{"sourceType":"arxiv-journal-reference","sourceUrl":"https://arxiv.org/abs/2607.08729","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"AVSS 2026","level":"strong-author-metadata"}]}],"capabilityGroups":["Multimodal Perception","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_vsro-200_f4255395","familyId":"bmf_76014c7782c1","name":"VSRo-200","oneLine":"A large-scale dataset for visual speech recognition in Romanian, with 200 hours of video and annotations. It studies supervision quality, robustness under domain shift, and multimodal fusion.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Robustness"],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.08112","pdf":"https://arxiv.org/pdf/2607.08112","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.08112"},"evidence":{"snippet":"Building on this dataset, we establish a benchmark for visual speech recognition in low-resource settings.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08112"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A large-scale dataset for visual speech recognition in Romanian, with 200 hours of video and annotations. It studies supervision quality, robustness under domain shift, and multimodal fusion.","whyItMatters":"The dataset enables research in low-resource visual speech recognition, but the paper primarily focuses on studying supervision and robustness rather than defining a fixed benchmark with a scoring contract. It is a dataset resource for training, not a standalone evaluation benchmark.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8ba8ecc8b6641fbc945e1839ad3d4dd577d9fdde76d84ce9b6f52041124f65c3"},"motivation":"We introduce VSRo-200, the first large-scale dataset for visual speech recognition (lip reading) in Romanian, comprising 200 hours of real-world podcast videos.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.08112","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_uniclawbench_4d04906d","familyId":"bmf_36cef9ecb058","name":"UniClawBench","oneLine":"A capability-driven benchmark for proactive agents in real-world tasks, with 400 bilingual tasks across five capabilities. It evaluates agents in Docker containers using step-by-step checkpoints and a closed-loop strategy with executor, supervisor, and user agents.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.08768","pdf":"https://arxiv.org/pdf/2607.08768","project":"https://uniclawbench.github.io","code":"https://github.com/HKU-MMLab/UniClawBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.08768"},"evidence":{"snippet":"To address these limitations, we introduce UniClawBench, the first capability-driven benchmark designed to evaluate proactive agents in dynamic, real-world settings.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":34,"hfDailySubmittedAt":null,"githubStars":38,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08768"},"ranking":{"90d":{"score":74,"rank":52,"coverage":0.7,"confidence":"Medium"}},"description":"A capability-driven benchmark for proactive agents in real-world tasks, with 400 bilingual tasks across five capabilities. It evaluates agents in Docker containers using step-by-step checkpoints and a closed-loop strategy with executor, supervisor, and user agents.","whyItMatters":"Existing agent benchmarks rely on sandboxed environments and single-turn paradigms, which do not reflect real-world complexity. This benchmark provides a dynamic, capability-based evaluation that helps compare models and agent frameworks, aiding in identifying failure root causes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2f779a2c4ca85aa66b2e5e837c8b79ccc20b727c7fcd6b6b7d04b6ff494ace17"},"motivation":"The rapid development of large language models and multimodal large language models has accelerated the emergence of proactive agents capable of operating everyday tools and assisting users in real-world environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.08768","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HKU-MMLab","organizationType":"academic-lab","sourceUrl":"https://github.com/HKU-MMLab/UniClawBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_solarchain-eval_aac1b95f","familyId":"bmf_b1b8147bf474","name":"SolarChain-Eval","oneLine":"A physics-constrained benchmark for evaluating economic agents in decentralized energy markets, with a Gymnasium-compatible MDP and an LLM-based Planner/Auditor layer. It evaluates market utility, physical safety, slippage, action smoothness, spatial fairness, and auditability.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.08681","pdf":"https://arxiv.org/pdf/2607.08681","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.08681"},"evidence":{"snippet":"Therefore, we propose SolarChain-Eval, a physics-constrained benchmark for evaluating trustworthy economic agents.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08681"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A physics-constrained benchmark for evaluating economic agents in decentralized energy markets, with a Gymnasium-compatible MDP and an LLM-based Planner/Auditor layer. It evaluates market utility, physical safety, slippage, action smoothness, spatial fairness, and auditability.","whyItMatters":"The benchmark addresses the need to evaluate agent trustworthiness in cyber-physical systems, where agents must balance utility with safety. However, without clear public access to the environment and scoring mechanisms, its standalone value is limited.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8fc582953143a68457dab901b4907c9f19bd313e45e09814a690f57a6be3dd5f"},"motivation":"As agentic AI systems are increasingly applied to cyber-physical environments, their evaluation requires assessment of both task performance and trustworthiness.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.08681","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_omnifood-bench_13bf03fa","familyId":"bmf_11629f46fc0a","name":"OmniFood-Bench","oneLine":"A benchmark for evaluating Vision-Language Models on nutrient reasoning and personalized health advice, with progressive capabilities: basic perception, quantitative reasoning, and safety-critical advisory.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.08423","pdf":"https://arxiv.org/pdf/2607.08423","project":"https://anonymous.4open.science/r/OmniFood-Bench-7D0B","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.08423"},"evidence":{"snippet":"In this paper, we introduce OmniFood-Bench, a comprehensive benchmark constructed from the MM-Food-100K dataset.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08423"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark for evaluating Vision-Language Models on nutrient reasoning and personalized health advice, with progressive capabilities: basic perception, quantitative reasoning, and safety-critical advisory.","whyItMatters":"The benchmark targets a critical gap in food systems AI evaluation, which often focuses on classification. By testing reasoning to safety-critical advice, it aims to establish standards for trustworthiness in public health applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e7ca01a5a8a80c7c669b4edbfae234abc7e00a4aa70688a205bea2131cd03686"},"motivation":"The rapid integration of Large Vision-Language Models (VLMs) into critical infrastructure promises to revolutionize personalized healthcare and dietary management.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.08423","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_multiview-bench_66d1791f","familyId":"bmf_6e0786a96b06","name":"MultiView-Bench","oneLine":"Evaluates multi-view integration in vision-language models using diagnostic tasks for 3D scene comprehension, with a fixed-view baseline and proposed ViewNavigator method.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.08970","pdf":"https://arxiv.org/pdf/2607.08970","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.08970"},"evidence":{"snippet":"We introduce MultiView-Bench, a diagnostic benchmark expressly designed to evaluate multi-view integration for holistic 3D scene comprehension.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08970"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Evaluates multi-view integration in vision-language models using diagnostic tasks for 3D scene comprehension, with a fixed-view baseline and proposed ViewNavigator method.","whyItMatters":"Addresses the gap in evaluating VLMs' ability to integrate observations across viewpoints into allocentric 3D models, a prerequisite for downstream tasks like assembly.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c5f573d94c580572008ee677d38da90719740e72a7bce3aa91d292e048c5ef15"},"motivation":"Recent benchmarks for VLMs largely assess single- or limited-view perception, leaving untested the core cognitive ability to integrate observations across viewpoints into a coherent, world-centric (allocentric) 3D mental model.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.08970","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_long-horizon-terminal-bench_593d14a6","familyId":"bmf_0d2d7122f15f","name":"Long-Horizon-Terminal-Bench","oneLine":"Evaluates long-horizon terminal tasks in a containerized environment with hidden verifiers and dense reward grading across 46 tasks and nine categories.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Multimodal","Code"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.08964","pdf":"https://arxiv.org/pdf/2607.08964","project":null,"code":"https://github.com/zli12321/LHTB","data":null,"hfPaper":"https://huggingface.co/papers/2607.08964"},"evidence":{"snippet":"We introduce Long-Horizon-Terminal-Bench, a terminal benchmark of 46 long-horizon tasks spanning nine categories, including experiment reproduction, software engineering, multimodal analysis, interactive games, and scientific computing.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":77,"hfDailySubmittedAt":null,"githubStars":624,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08964"},"ranking":{"90d":{"score":84,"rank":7,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates long-horizon terminal tasks in a containerized environment with hidden verifiers and dense reward grading across 46 tasks and nine categories.","whyItMatters":"Provides a more demanding evaluation for agentic long-horizon planning and partial credit, addressing gaps in existing terminal benchmarks that only measure final outcomes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"77d3230de58f2233270e143a2d5ffcbf256c0414dba28bafbacab8bdb5e5148b"},"motivation":"AI agents have become capable of autonomously completing short, well-specified tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.08964","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_include-bench_e92f5105","familyId":"bmf_8801a3a5bd87","name":"INCLUDE-BENCH","oneLine":"Evaluates disability-related bias in text-to-image models using 119K generated images across bias dimensions and contexts, with the Stereotype Content Model Score.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.08515","pdf":"https://arxiv.org/pdf/2607.08515","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.08515"},"evidence":{"snippet":"To address this, we introduce INCLUDE-BENCH, the first large-scale benchmark for evaluating disability-related bias in T2I models.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08515"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates disability-related bias in text-to-image models using 119K generated images across bias dimensions and contexts, with the Stereotype Content Model Score.","whyItMatters":"Addresses the underexplored area of disability stereotypes in T2I models, providing a large-scale evaluation for representational harms.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"37e065f941ad815e9e2913dfd28709c6916cf52b0f3f3d23d44c087eed4208a1"},"motivation":"Text-to-image (T2I) models have been shown to exhibit social biases.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.08515","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_ideagene-bench_3d711bf3","familyId":"bmf_04604847c861","name":"IdeaGene-Bench","oneLine":"Evaluates scientific lineage reasoning and lineage-grounded idea generation through two tracks: closed-form IG-Exam and generation IG-Arena with Population-Evolution Score.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.08758","pdf":"https://arxiv.org/pdf/2607.08758","project":null,"code":"https://github.com/VisionXLab/IdeasHaveGenomes","data":null,"hfPaper":"https://huggingface.co/papers/2607.08758"},"evidence":{"snippet":"We present IdeaGene-Bench (IG-Bench), a benchmark for scientific lineage reasoning and lineage-grounded idea generation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":41,"hfDailySubmittedAt":null,"githubStars":32,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08758"},"ranking":{"90d":{"score":73,"rank":55,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates scientific lineage reasoning and lineage-grounded idea generation through two tracks: closed-form IG-Exam and generation IG-Arena with Population-Evolution Score.","whyItMatters":"Fills the gap in evaluating AI systems' understanding of scientific idea evolution, with results showing a compositional bottleneck in current models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"65157e110ddb54f4fe771e238bef226360b968653ef0c8afe82164cac8faaef1"},"motivation":"Scientific ideas rarely start from a blank page.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.08758","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"VisionXLab","organizationType":"academic-lab","sourceUrl":"https://github.com/VisionXLab/IdeasHaveGenomes","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_dexverse_baeb3cfa","familyId":"bmf_e243a022bf00","name":"DexVerse","oneLine":"Evaluates dexterous manipulation across 100 tasks, multiple embodiments, and visual variations, with 3,180 demonstrations and a VR teleoperation interface.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.08751","pdf":"https://arxiv.org/pdf/2607.08751","project":"https://ycyao216.github.io/DexVerse.site","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.08751"},"evidence":{"snippet":"We present DexVerse, a large-scale and modular benchmark for dexterous manipulation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08751"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Evaluates dexterous manipulation across 100 tasks, multiple embodiments, and visual variations, with 3,180 demonstrations and a VR teleoperation interface.","whyItMatters":"Provides a comprehensive testbed for studying cross-task and cross-embodiment generalization in dexterous manipulation, addressing gaps in existing benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f915a8df24c6c7054ed0e481048ad2cb383cd9cdc5fd6a895172af61f1d8ada1"},"motivation":"Building general-purpose dexterous manipulation policies requires benchmarks that go beyond isolated tasks to systematically evaluate policies across diverse interaction modes, sensory conditions, and robot embodiments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.08751","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_causalds_e9c95fdf","familyId":"bmf_c55169b2a8eb","name":"CausalDS","oneLine":"Evaluates causal reasoning in data-science workflows using synthetic scenes with hidden structural causal models, covering Pearl's three rungs and abstention scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.08093","pdf":"https://arxiv.org/pdf/2607.08093","project":null,"code":"https://github.com/andleb/causalds","data":null,"hfPaper":"https://huggingface.co/papers/2607.08093"},"evidence":{"snippet":"We introduce CausalDS, a benchmark for evaluating causal reasoning in agentic data-science workflows.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":5,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08093"},"ranking":{"90d":{"score":46,"rank":227,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates causal reasoning in data-science workflows using synthetic scenes with hidden structural causal models, covering Pearl's three rungs and abstention scoring.","whyItMatters":"Bridges symbolic causal reasoning and realistic data analysis, providing a joint evaluation of reasoning, tool use, and uncertainty quantification in agentic settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"63fa98bc91ff1b80f07a3208683bc0c8a0b01b2787fa430a529bfa0994787fbf"},"motivation":"Large language models (LLMs) increasingly act as integrated data-science agents, combining abstract reasoning with advanced tool use.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.08093","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_blind-spots-bench_1b74311c","familyId":"bmf_74da405a337c","name":"Blind-Spots-Bench","oneLine":"Evaluates reasoning blind spots in language, vision-language, and image-generation models across 235 samples with structured reference solutions and taxonomy.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.08317","pdf":"https://arxiv.org/pdf/2607.08317","project":null,"code":"https://github.com/matteosantelmo/reasoning-blind-spots","data":null,"hfPaper":"https://huggingface.co/papers/2607.08317"},"evidence":{"snippet":"We introduce $\\texttt{blind-spots-bench}$, a benchmark designed to expose such blind spots through tasks that appear simple for humans but remain challenging for modern AI.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":37,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.08317"},"ranking":{"90d":{"score":55,"rank":164,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates reasoning blind spots in language, vision-language, and image-generation models across 235 samples with structured reference solutions and taxonomy.","whyItMatters":"Serves as a diagnostic stress test exposing tasks that humans find easy but AI models struggle with, highlighting gaps not captured by existing benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2ee38fb7fce4dca5a602a776c08b2ff548172b2b38b3d879b249f7720491acf9"},"motivation":"Modern AI models achieve strong performance on many established benchmarks, yet they still fail on tasks that humans find almost trivial, such as manipulating a string or drawing a dog with five legs.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.08317","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_synthave_eb33ab8a","familyId":"bmf_73cc3ce2801f","name":"SynthAVE","oneLine":"Presents a synthetic labeling pipeline for e-commerce attribute extraction with human validation via multi-LLM arena voting.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.07469","pdf":"https://arxiv.org/pdf/2607.07469","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.07469"},"evidence":{"snippet":"We present SynthAVE, a large-scale human-validated benchmark for attribute value extraction spanning 12,726 products across 229 product types, 792 attributes, and 4 languages (Spanish, French, Italian, German).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.07469"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Presents a synthetic labeling pipeline for e-commerce attribute extraction with human validation via multi-LLM arena voting.","whyItMatters":"Demonstrates a cost-effective method for large-scale label generation with quality control, but is not a benchmark for model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"59716af9ba809dfe7a58a2b2996cf7370992c87378d8b6d33736185e6cc821ad"},"motivation":"Fine-tuning large language models (LLMs) for e-commerce attribute extraction requires labeled data representative across thousands of product types, attributes, and multiple languages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.07469","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_shift-drift_175f8c38","familyId":"bmf_1080f520bb7a","name":"Shift & Drift","oneLine":"Shift & Drift evaluates closed-loop motion planners on two tracks: Semantic Shift, which uses a conversion pipeline to transform the DeepScenario Open 3D dataset into nuPlan for zero-shot testing on 1,182 scenarios across German cities and San Francisco, and State-Distribution Drift, which injects stochastic perturbations into ego-vehicle dynamics. Scoring is based on safety and progress metrics.","area":"Vision & 3D","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Automotive"],"capabilities":["Planning","Geometric reasoning"],"topics":["cs.RO"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.07844","pdf":"https://arxiv.org/pdf/2607.07844","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.07844"},"evidence":{"snippet":"To address this, we present Shift & Drift, a novel dual-track benchmark designed to rigorously stress-test motion planners across two critical axes of distribution shift: (1) The Semantic Shift Track leverages a novel conversion pipeline that transforms the aerial, DeepScenario Open 3D dataset into the nuPlan simulation framework.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.07844"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Shift & Drift evaluates closed-loop motion planners on two tracks: Semantic Shift, which uses a conversion pipeline to transform the DeepScenario Open 3D dataset into nuPlan for zero-shot testing on 1,182 scenarios across German cities and San Francisco, and State-Distribution Drift, which injects stochastic perturbations into ego-vehicle dynamics. Scoring is based on safety and progress metrics.","whyItMatters":"Addresses the evaluation gap in generalization of motion planners to novel urban topologies and robustness to execution perturbations, providing a dual-track benchmark that quantifies the trade-off between imitation fidelity and closed-loop resilience.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"874d16e832f1ac20704d988aa859b1d5a60fb347a51d0047854909f0e8199d99"},"motivation":"While closed-loop motion planners trained on large-scale, object-level datasets, e.g., nuPlan, demonstrate strong in-distribution (ID) performance, their generalization to novel urban topologies and recovery mechanisms following execution perturbations remain under-explored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"2026 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS 2026)","evidence":"Accepted at 2026 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS 2026)","evidenceUrl":"https://arxiv.org/abs/2607.07844","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"2026 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS 2026)","reviewStatus":"accepted","decisionRaw":"Accepted at 2026 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS 2026)","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.07844","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at 2026 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS 2026)","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_phitsbench_3767e77d","familyId":"bmf_a8ec597bae7e","name":"PHITSBench","oneLine":"PHITSBench evaluates AI-assisted generation of PHITS radiation-transport input via natural language across 282 tasks in three workflows: Edit, Repair, and Reproduce. Scoring uses a Composite Metric Score combining execution success and agreement with reference transport observables.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.09789","pdf":"https://arxiv.org/pdf/2607.09789","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.09789"},"evidence":{"snippet":"We introduce PHITSBench, an execution-scored benchmark for the Monte Carlo Particle and Heavy Ion Transport code System (PHITS).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.09789"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PHITSBench evaluates AI-assisted generation of PHITS radiation-transport input via natural language across 282 tasks in three workflows: Edit, Repair, and Reproduce. Scoring uses a Composite Metric Score combining execution success and agreement with reference transport observables.","whyItMatters":"Provides an execution-grounded benchmark for a niche task, highlighting the need for machine-readable knowledge bases and curated training data in AI-assisted radiation-transport modeling.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"71f35ffb236e6c3b942ee8da7e1ccfc29ca5c8e572bf78082808b3de5eecf18d"},"motivation":"We introduce PHITSBench, an execution-scored benchmark for the Monte Carlo Particle and Heavy Ion Transport code System (PHITS).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.09789","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_perfopt-bench_3a913e78","familyId":"bmf_1425682db99d","name":"PERFOPT-Bench","oneLine":"PERFOPT-Bench evaluates coding agents on software performance optimization tasks, requiring profiling, diagnosing bottlenecks, editing code while preserving correctness, and verifying reproducible speedups. Scoring includes hidden correctness tests, verified-speedup measurement, and trajectory-level audit.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.07744","pdf":"https://arxiv.org/pdf/2607.07744","project":"https://anonymous.4open.science/r/Dataset-D3CC","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.07744"},"evidence":{"snippet":"We introduce PERFOPT-Bench, a benchmark for evaluating this full performance-engineering loop.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.07744"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"PERFOPT-Bench evaluates coding agents on software performance optimization tasks, requiring profiling, diagnosing bottlenecks, editing code while preserving correctness, and verifying reproducible speedups. Scoring includes hidden correctness tests, verified-speedup measurement, and trajectory-level audit.","whyItMatters":"Fills the gap in benchmarks focusing on performance engineering rather than functional correctness, measuring practical speedups on real execution targets and addressing issues like shortcut exploitation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d23f95d9142ead8dcf00fe5ed0553c972cf1a01323de436ebea31c7b3f7d2da7"},"motivation":"Coding-agent benchmarks have largely measured whether agents can produce functionally correct patches, but production software also demands measurable speedups on real execution targets.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.07744","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_occur-bench_c8c0341e","familyId":"bmf_e63aa94e3c0e","name":"OCCUR-Bench","oneLine":"OCCUR-Bench evaluates temporal preservation in conversational image editing, providing occlusion-and-revelation scenarios with historical restoration references to assess faithful restoration of occluded content.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.07051","pdf":"https://arxiv.org/pdf/2607.07051","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.07051"},"evidence":{"snippet":"We introduce OCCUR-Bench, a diagnostic benchmark for temporal preservation in conversational image editing.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.07051"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OCCUR-Bench evaluates temporal preservation in conversational image editing, providing occlusion-and-revelation scenarios with historical restoration references to assess faithful restoration of occluded content.","whyItMatters":"Targets the under-explored problem of preserving content that temporarily disappears during multi-turn editing, offering a diagnostic benchmark for faithful restoration.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"733c62fcbb9583195c9205d3a59310c787577327fb0281b032ac1bdd23fbc8bc"},"motivation":"Conversational image editing requires preserving not only visible content, but also content that temporarily disappears across turns.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.07051","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mira-math_da1f22ae","familyId":"bmf_543fc16ad0c8","name":"MIRA-Math","oneLine":"MIRA-Math evaluates mathematical reasoning where each problem is missing exactly one necessary atomic fact that must be requested in natural language under a strict budget, then integrated into an exact answer. It contains 2,310 instances across 22 typed mathematical families.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.07391","pdf":"https://arxiv.org/pdf/2607.07391","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.07391"},"evidence":{"snippet":"We introduce MIRA-Math, a benchmark for a narrower diagnostic capability: solving mathematical problems whose full latent state has a unique answer, but whose solver-facing view is missing exactly one necessary atomic fact.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.07391"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MIRA-Math evaluates mathematical reasoning where each problem is missing exactly one necessary atomic fact that must be requested in natural language under a strict budget, then integrated into an exact answer. It contains 2,310 instances across 22 typed mathematical families.","whyItMatters":"Isolates the diagnostic capability of minimal information requesting separate from broader tool use or long-horizon dialogue, revealing that request success and final-answer accuracy are separable.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d5082b9d9e03d5bd261cbd50436766fa6cb04e245b67357bddc3105979bc4ad1"},"motivation":"Mathematical reasoning benchmarks typically provide all facts needed to solve each problem, while interactive benchmarks often mix reasoning with tools, retrieval, and long-horizon dialogue.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.07391","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_madb_64f075cb","familyId":"bmf_24baaf7d2a98","name":"MADB","oneLine":"MADB is a large-scale dataset and benchmark for music aesthetic assessment, comprising 9,999 tracks annotated by 30 trained annotators across 10 perceptual dimensions and an overall score, with textual comments. It includes a unified evaluation framework over multiple pretrained models.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SD"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.06929","pdf":"https://arxiv.org/pdf/2607.06929","project":null,"code":"https://github.com/knownree/madb","data":null,"hfPaper":"https://huggingface.co/papers/2607.06929"},"evidence":{"snippet":"We introduce MADB, a large-scale dataset and benchmark comprising 9,999 tracks annotated by 30 trained annotators.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06929"},"ranking":{"90d":{"score":46,"rank":220,"coverage":0.55,"confidence":"Low"}},"description":"MADB is a large-scale dataset and benchmark for music aesthetic assessment, comprising 9,999 tracks annotated by 30 trained annotators across 10 perceptual dimensions and an overall score, with textual comments. It includes a unified evaluation framework over multiple pretrained models.","whyItMatters":"Provides structured aesthetic annotations for a previously underexplored area, enabling measurement of model-human gaps in music understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9f80e509c552345f5b19ac57e8de74e1261ad68cc4d2df8aeb181a3b0024b086"},"motivation":"Music aesthetic assessment is a challenging yet underexplored problem, requiring models to capture fine-grained, multi-dimensional human perceptual judgments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06929","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Knownree","organizationType":"academic-lab","sourceUrl":"https://github.com/knownree/madb","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_kotlin-benchmark_jetbrains-2026","familyId":"bmf_11a0a2a20c95","name":"Kotlin Benchmark","oneLine":"Evaluates AI coding agents on real-world Kotlin tasks from nine open-source repositories. The benchmark uses reproducible Docker environments, regression tests, and a scoring protocol that awards pass only when all expected test transitions are met.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation","Software engineering"],"topics":["Coding Agents","Kotlin"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://blog.jetbrains.com/kotlin/2026/07/introducing-the-kotlin-benchmark-evaluate-ai-coding-agents-on-real-world-kotlin-tasks/","pdf":null,"project":"https://kotlinlang.org/benchmark/","code":"https://github.com/Kotlin/kotlin-swe-bench","data":null,"hfPaper":null},"evidence":{"snippet":"The Kotlin Benchmark evaluates AI coding agents on real-world Kotlin tasks using reproducible execution and regression tests.","reasonCodes":["official project release","reviewed non-arXiv source"]},"dataStatus":"primary-source-reviewed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":50,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"official-project","id":"jetbrains-kotlin-benchmark-2026"},"ranking":{"90d":{"score":71,"rank":71,"coverage":0.55,"confidence":"Low"}},"description":"Evaluates AI coding agents on real-world Kotlin tasks from nine open-source repositories. The benchmark uses reproducible Docker environments, regression tests, and a scoring protocol that awards pass only when all expected test transitions are met.","whyItMatters":"This benchmark addresses the lack of reference evaluation for Kotlin-specific coding agents, providing a reproducible, task-level framework for comparing agent performance on real-world Kotlin issues and tracking progress.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-22T18:51:01.308665Z","inputHash":"12dd9616dd867f1805bf6e8da925809f8216ad775f20f690a97b3c405f36d32d"},"motivation":"Measure whether general coding agents can solve realistic Kotlin repository issues under a reproducible SWE-bench-style protocol.","constructionDetail":"JetBrains adapted the SWE-bench methodology to resolved issues from real Kotlin repositories, packaged with Harbor-compatible containers and regression tests.","metrics":[{"name":"Tasks","value":"105","note":"initial release snapshot"},{"name":"Repositories","value":"8","note":"initial release snapshot"}],"curation":{"state":"source-reviewed","reviewedAt":"2026-08-19","sources":["https://blog.jetbrains.com/kotlin/2026/07/introducing-the-kotlin-benchmark-evaluate-ai-coding-agents-on-real-world-kotlin-tasks/","https://kotlinlang.org/benchmark/methodology/","https://github.com/Kotlin/kotlin-swe-bench"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://blog.jetbrains.com/kotlin/2026/07/introducing-the-kotlin-benchmark-evaluate-ai-coding-agents-on-real-world-kotlin-tasks/","source":"official-project","evidenceLevel":"official","verifiedAt":"2026-08-19T00:00:00Z"},"releaseDates":{"firstPublicAt":"2026-07-08","paperV1At":null},"publishers":[{"name":"JetBrains","organizationType":"company-research-lab","sourceUrl":"https://github.com/Kotlin/kotlin-swe-bench","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_imagingbench_e9f34716","familyId":"bmf_960f5481dc88","name":"ImagingBench","oneLine":"ImagingBench evaluates agentic AI systems on 20 computational imaging tasks spanning ray and wave optics, image signal processing, inverse reconstruction, computational sensing, and calibration, across three settings: Expert, Planner, and Forward.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.07189","pdf":"https://arxiv.org/pdf/2607.07189","project":"https://cirp-lab.github.io/imagingbench","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.07189"},"evidence":{"snippet":"We present ImagingBench, a benchmark of 20 computational imaging tasks spanning five categories: ray and wave optics, image signal processing, inverse reconstruction, computational sensing, and calibration.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.07189"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ImagingBench evaluates agentic AI systems on 20 computational imaging tasks spanning ray and wave optics, image signal processing, inverse reconstruction, computational sensing, and calibration, across three settings: Expert, Planner, and Forward.","whyItMatters":"Reveals the gap between semantic visual competence and physically grounded imaging performance, providing a unified testbed to measure progress in agentic AI for computational imaging.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4f4a63ed0f2afe3e4c0f49b95c07e0baf417a36737e3d86c1dfb8388b35a0dcd"},"motivation":"Vision-language models (VLMs) and agentic AI have shown strong performance on semantic visual tasks, but it remains unclear whether they can handle the physics and inverse problems that underlie computational imaging.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.07189","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_freightbidbench_ee372bca","familyId":"bmf_a92125a5e129","name":"FreightBidBench","oneLine":"FreightBidBench is a public-calibrated, closed-loop benchmark for real-time truckload bid acceptance with explicit operational feasibility and economics, including pickup reach, appointment windows, hours-of-service, and yard delays. It provides two full-horizon hindsight ceilings for evaluation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.07343","pdf":"https://arxiv.org/pdf/2607.07343","project":null,"code":"https://github.com/aswincsekar/freightbidbench","data":null,"hfPaper":"https://huggingface.co/papers/2607.07343"},"evidence":{"snippet":"We introduce FreightBidBench, a public-calibrated, dependency-free, closed-loop benchmark in which feasibility (pickup reach, appointment windows, simplified hours-of-service, stochastic yard delays) and economics (service-failure penalty, terminal fleet value, daily price-premium window) are explicit, versioned, and reproducible from public Freight Analysis Framework and U.S.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.07343"},"ranking":{"90d":{"score":29,"rank":369,"coverage":0.55,"confidence":"Low"}},"description":"FreightBidBench is a public-calibrated, closed-loop benchmark for real-time truckload bid acceptance with explicit operational feasibility and economics, including pickup reach, appointment windows, hours-of-service, and yard delays. It provides two full-horizon hindsight ceilings for evaluation.","whyItMatters":"Offers a reproducible benchmark for a dynamic stochastic decision problem that lacked public options, enabling evaluation of bid acceptance policies under operational constraints.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"80bbb71340db600bbf218cb10a7f2743c2adc67f4834693e9a0dd3ebb760ec3e"},"motivation":"Online truckload bid acceptance is a closed-loop stochastic decision problem in which a carrier or broker must, in real time, accept or reject a tendered load subject to operational feasibility, fleet repositioning costs, and opportunity cost against future demand.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.07343","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Aswin C. Sekar","organizationType":"academic-lab","sourceUrl":"https://github.com/aswincsekar/freightbidbench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_deepswe_ebdcd38e","familyId":"bmf_3f3a44e7f6ef","name":"DeepSWE","oneLine":"Evaluates coding agents on 113 original, long-horizon software engineering tasks across 91 open-source repositories in five languages. Tasks are written from scratch, with hand-written verifiers that check requested functionality and accept any correct implementation.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.07946","pdf":"https://arxiv.org/pdf/2607.07946","project":"https://deepswe.datacurve.ai/","code":"https://github.com/datacurve-ai/deep-swe","data":"https://huggingface.co/datasets/datacurve/deep-swe","hfPaper":"https://huggingface.co/papers/2607.07946"},"evidence":{"snippet":"We release the benchmark, its verifiers, and the full record of evaluation trajectories.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-reviewed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":1478,"githubScope":"benchmark_repo","hfDatasetDownloads":918,"hfDatasetLikes":62},"source":{"type":"arxiv","id":"2607.07946"},"ranking":{"90d":{"score":83,"rank":9,"coverage":1.0,"confidence":"High","datasetDownloadRank":9,"datasetRankPopulation":51}},"description":"Evaluates coding agents on 113 original, long-horizon software engineering tasks across 91 open-source repositories in five languages. Tasks are written from scratch, with hand-written verifiers that check requested functionality and accept any correct implementation.","whyItMatters":"Addresses the gap of benchmarks relying on mined fixes and inherited tests, which can overstate model capability due to pretraining exposure and rigid grading. Provides a reusable evaluation path with verifiers and trajectories for assessing genuine problem-solving ability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e890c3ad2a987f7362993def072c47c833df4d1ceec4c5eb76afd49724188295"},"motivation":"DeepSWE is a benchmark of 113 original, long-horizon software engineering tasks for evaluating coding agents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"source-reviewed","reviewedAt":"2026-08-19","sources":["https://datacurve.ai/research","https://deepswe.datacurve.ai/","https://github.com/datacurve-ai/deep-swe","https://arxiv.org/abs/2607.07946"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.07946","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"releaseDates":{"firstPublicAt":"2026-05-18","paperV1At":"2026-07-08"},"publishers":[{"name":"DataCurve","organizationType":"company-research-lab","sourceUrl":"https://deepswe.datacurve.ai/","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_cann-bench_f89ecd9f","familyId":"bmf_521039f15a2e","name":"CANN Bench","oneLine":"Proposes a benchmark for AI-generated operator code on Huawei's Ascend NPU, covering 53 operators and 1060 test cases. Evaluation uses a three-dimensional weighted composite score for compilation, functional correctness, and performance.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20518","pdf":"https://arxiv.org/pdf/2607.20518","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20518"},"evidence":{"snippet":"We present CANN Bench, an open benchmark for AI-generated operator code on Huawei's Ascend NPU.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20518"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Proposes a benchmark for AI-generated operator code on Huawei's Ascend NPU, covering 53 operators and 1060 test cases. Evaluation uses a three-dimensional weighted composite score for compilation, functional correctness, and performance.","whyItMatters":"Could fill a gap in evaluating kernel-generation agents beyond CUDA/Triton, but clear artifact availability and a usable scoring contract are not confirmed from provided evidence.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"78b1e5069750a99098f386f8e76081436b294d5dbfa338943023ca32e02512a4"},"motivation":"AI agents are now capable of writing, compiling, and iteratively optimizing low-level operator kernels on different hardware platforms.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20518","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_webretriever_b067311d","familyId":"bmf_e07e1bfcf561","name":"WebRetriever","oneLine":"Introduces a benchmark with 800 websites and 1,550 tasks for web agent evaluation, plus the NavEval LLM-as-Judge framework and three evaluation protocols.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.06118","pdf":"https://arxiv.org/pdf/2607.06118","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.06118"},"evidence":{"snippet":"To address these limitations, we introduce WebRetriever, a large-scale benchmark encompassing 800 websites and 1,550 tasks across diverse domains, including consumer, professional, and enterprise sectors, with comprehensive coverage of user intent patterns.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06118"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Introduces a benchmark with 800 websites and 1,550 tasks for web agent evaluation, plus the NavEval LLM-as-Judge framework and three evaluation protocols.","whyItMatters":"Could offer large-scale cross-domain assessment for web agents, but the evaluation methodology relies on LLM-as-Judge and lacks clear public implementation details.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"eccbb4acb7b58abcaf0259cf698d04a1a0c929277566b685b603967157e82e82"},"motivation":"As web agents increasingly demonstrate capabilities in automated task execution, the development of robust evaluation frameworks for assessing their navigation and task completion performance has emerged as a critical research priority.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06118","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_vendorbench-100_8cfd26ec","familyId":"bmf_4b8f07b501cd","name":"VendorBench-100","oneLine":"Evaluates deepfake image detectors across three paradigms—commercial APIs, vision LLMs, and open-source detectors—on a fixed 100-image adversarial corpus. Uses a unified output schema and scores primarily by Matthews correlation coefficient with ROC-AUC.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.06254","pdf":"https://arxiv.org/pdf/2607.06254","project":null,"code":"https://github.com/sharayu-20/vendorbench-100","data":null,"hfPaper":"https://huggingface.co/papers/2607.06254"},"evidence":{"snippet":"We introduce VendorBench-100, a cross-paradigm benchmark that evaluates 36 representative models using a single adversarial 100-image corpus, a unified output schema, and a common evaluation framework.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06254"},"ranking":{"90d":{"score":39,"rank":289,"coverage":0.55,"confidence":"Low"}},"description":"Evaluates deepfake image detectors across three paradigms—commercial APIs, vision LLMs, and open-source detectors—on a fixed 100-image adversarial corpus. Uses a unified output schema and scores primarily by Matthews correlation coefficient with ROC-AUC.","whyItMatters":"Provides a common ground for comparing disparate detector types, addressing the lack of unified evaluation. Identifies metric correlation and calibration issues that matter for real-world deployment decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bd1b8e98e3b7544d6982fbf9b2e6e71e9d058d53e37115bd6d2068ad8605b349"},"motivation":"Deepfake image detection is served by three fundamentally different paradigms - commercial APIs, zero-shot vision-language models (LLMs), and open-source detectors - that are rarely evaluated under a common protocol, making direct comparison difficult.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06254","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Sharayu Deshmukh","organizationType":"community","sourceUrl":"https://github.com/sharayu-20/vendorbench-100","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ui2app_cf415439","familyId":"bmf_aca78e2fdb0c","name":"UI2App","oneLine":"Benchmarks visual interaction inference in executable web application generation. Contains 327 screenshots in 45 sets, evaluating executability, navigation reachability, visual fidelity, and interaction inference via the IIS metric.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.06306","pdf":"https://arxiv.org/pdf/2607.06306","project":null,"code":"https://github.com/chenmancm169/UI2App","data":null,"hfPaper":"https://huggingface.co/papers/2607.06306"},"evidence":{"snippet":"To address this gap, we introduce UI2App, the first benchmark targeting interaction inference, the ability to recover application behavior from screenshots alone, without any textual or behavioral guidance.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06306"},"ranking":{"90d":{"score":48,"rank":212,"coverage":0.7,"confidence":"Medium"}},"description":"Benchmarks visual interaction inference in executable web application generation. Contains 327 screenshots in 45 sets, evaluating executability, navigation reachability, visual fidelity, and interaction inference via the IIS metric.","whyItMatters":"Targets a gap in measuring behavior inference from screenshots, not just visual fidelity. Could help assess models' ability to produce interactive applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1ddac74dcec827323df25757304b635430681d503488d8ea5f382cda06cfa635"},"motivation":"Large language models (LLMs) have demonstrated growing competence in web page generation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06306","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_thorarena_63609675","familyId":"bmf_bc2239e32cf5","name":"ThorArena","oneLine":"Presents a benchmark for force-aware humanoid interaction using demonstrations with synchronized motion and force data. Includes Force-Aware Tracking Score and a simulation protocol.","area":"Robotics & Embodied AI","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.RO"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.06052","pdf":"https://arxiv.org/pdf/2607.06052","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.06052"},"evidence":{"snippet":"In this paper, we present ThorArena, a benchmark for evaluating force-aware humanoid interaction based on human demonstrations with synchronized motion and force measurements.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06052"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Presents a benchmark for force-aware humanoid interaction using demonstrations with synchronized motion and force data. Includes Force-Aware Tracking Score and a simulation protocol.","whyItMatters":"Could fill a gap in evaluating humanoid control under physical interaction, which is often overlooked. But unclear evaluation environment and public availability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"be9d6fd0375cd1a2e313a170ae97922221dc5b75fa5f846353474380a1063886"},"motivation":"Humanoid robots are increasingly expected to perform contact-rich tasks that require not only accurate whole-body motion but also robust physical interaction with surrounding objects and humans.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06052","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"general"},{"id":"bm_spider-2-0-aifunc_d9873e24","familyId":"bmf_020b339c7771","name":"Spider 2.0-AIFunc","oneLine":"Extends text-to-SQL to AI-native SQL workflows with 465 instances across 125 real-world databases on Snowflake. Tasks require using AI functions like classification and sentiment analysis. Evaluates execution accuracy.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.06229","pdf":"https://arxiv.org/pdf/2607.06229","project":null,"code":"https://github.com/Leolty/Spider2-AIFunc","data":null,"hfPaper":"https://huggingface.co/papers/2607.06229"},"evidence":{"snippet":"We introduce Spider 2.0-AIFunc, a benchmark of 465 verified instances across 125 real-world databases covering six types of AI functions on the Snowflake platform.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06229"},"ranking":{"90d":{"score":39,"rank":269,"coverage":0.7,"confidence":"Medium"}},"description":"Extends text-to-SQL to AI-native SQL workflows with 465 instances across 125 real-world databases on Snowflake. Tasks require using AI functions like classification and sentiment analysis. Evaluates execution accuracy.","whyItMatters":"Addresses the gap of benchmarks not covering AI-native SQL capabilities that are increasingly available in cloud platforms. Provides a reusable dataset and evaluation harness for a new task type.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b4de75b68993207d60c9728ecee5c188aba765d4980f4e90b8594f79cfab9fa0"},"motivation":"Major cloud data platforms now expose large language model capabilities as native SQL functions, enabling analysts to perform classification, filtering, sentiment analysis, extraction, similarity search, and aggregation within ordinary SQL queries.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06229","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Leolty","organizationType":"community","sourceUrl":"https://github.com/Leolty/Spider2-AIFunc","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_rubench_84878dc8","familyId":"bmf_8cbfda054603","name":"RuBench","oneLine":"Evaluates coding agents on 25 repository-level tasks in Russian, mined from recent fix commits across five open-source projects. Graded by upstream regression tests with withheld oracles. Multiple rounds document model change and contamination audits.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.06411","pdf":"https://arxiv.org/pdf/2607.06411","project":null,"code":"https://github.com/eugeneshilow/rubench","data":null,"hfPaper":"https://huggingface.co/papers/2607.06411"},"evidence":{"snippet":"We introduce RuBench 1.0, a benchmark of 25 tasks mined from recent fix commits in five live open-source repositories (aiohttp, aiogram, Laravel, NestJS, Fastify), each specified natively in Russian -- written from scratch, not translated -- and judged by the upstream maintainer's regression tests, which we withhold from release.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06411"},"ranking":{"90d":{"score":42,"rank":255,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates coding agents on 25 repository-level tasks in Russian, mined from recent fix commits across five open-source projects. Graded by upstream regression tests with withheld oracles. Multiple rounds document model change and contamination audits.","whyItMatters":"Provides a benchmark with natively authored non-English specifications, addressing a gap in multilingual agent evaluation. Includes rigorous auditing and honest scores, which matter for reliability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a9be3960d9a6af537aba4350cba5c298e1b5bd51dd87ec7ba0a52f8444c6a29e"},"motivation":"Developers increasingly delegate real maintenance work to product-grade coding agents, and many state tasks in their native language, in the style of a customer request rather than a curated English issue.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06411","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Evgeny Shilov","organizationType":"community","sourceUrl":"https://github.com/eugeneshilow/rubench","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_polyworkbench_2abf54e5","familyId":"bmf_91789172102b","name":"PolyWorkBench","oneLine":"PolyWorkBench evaluates LLM agents on multilingual, long-horizon workplace workflows across five domains: commerce, knowledge work, legal analysis, localization, and manufacturing. It includes 67 tasks, structured scoring via Grade, executable state verification with Pytest, and LLM-as-Judge diagnostics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.06008","pdf":"https://arxiv.org/pdf/2607.06008","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.06008"},"evidence":{"snippet":"We introduce PolyWorkBench, a benchmark designed to evaluate LLM agents on multilingual, long-horizon workplace workflows.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06008"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"PolyWorkBench evaluates LLM agents on multilingual, long-horizon workplace workflows across five domains: commerce, knowledge work, legal analysis, localization, and manufacturing. It includes 67 tasks, structured scoring via Grade, executable state verification with Pytest, and LLM-as-Judge diagnostics.","whyItMatters":"This benchmark fills the gap of evaluating LLM agents on tasks that combine multilinguality and long-horizon execution, providing a structured protocol to compare agent performance and identify systematic failure modes in cross-lingual scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"40e85edc51a0543817b5765dc2b8757fba2da41c8f8d09fa6b86e7a1e309dcee"},"motivation":"While Large Language Model (LLM) agents excel at monolingual long-horizon planning and tool use, enterprise workflows inherently require processing multilingual resources across extended trajectories.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06008","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"PolyWorkBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.06008","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_policyshiftbench_8ca9566e","familyId":"bmf_1a011cbd85dd","name":"PolicyShiftBench","oneLine":"PolicyShiftBench evaluates policy-adaptive image guardrailing: given an image and a current policy, a model must output a pass/block decision plus optional violated category IDs. The benchmark comprises 2,000 policy-discriminative instances over 265 images, each paired with multiple policy-conditioned prompts. Scoring uses binary pass/block accuracy and category attribution metrics.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.05910","pdf":"https://arxiv.org/pdf/2607.05910","project":null,"code":"https://github.com/ssmisya/PolicyShiftGuard","data":null,"hfPaper":"https://huggingface.co/papers/2607.05910"},"evidence":{"snippet":"We introduce PolicyShiftBench, a comprehensive benchmark with 2,000 policy-discriminative instances over 265 images, where each image is paired with 7.55 policy-conditioned prompts on average to test whether models adapt to the active policy rather than relying on image-level safety priors.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":38,"hfDailySubmittedAt":null,"githubStars":22,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.05910"},"ranking":{"90d":{"score":70,"rank":74,"coverage":0.7,"confidence":"Medium"}},"description":"PolicyShiftBench evaluates policy-adaptive image guardrailing: given an image and a current policy, a model must output a pass/block decision plus optional violated category IDs. The benchmark comprises 2,000 policy-discriminative instances over 265 images, each paired with multiple policy-conditioned prompts. Scoring uses binary pass/block accuracy and category attribution metrics.","whyItMatters":"Existing image safety benchmarks assume safety is a fixed property of an image, whereas real deployments vary policies across products and regions. PolicyShiftBench measures whether models can bind image evidence to the active policy rather than relying on image-level priors, providing a practical evaluation for guardrails in dynamic policy environments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"96af7e72f658f21967c3d891f085a7c6c3e49392ed4692678485a7b50b0f5cd2"},"motivation":"Image guardrails are typically trained and evaluated under a fixed safety policy, implicitly treating safety as an intrinsic property of an image.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.05910","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_pipbench_0f193694","familyId":"bmf_76865cb5cf7c","name":"PIPBench","oneLine":"PIPBench evaluates personalized image generation, where models must align outputs with a user's implicit visual preferences based on a few historically preferred images and a short prompt. It includes real-user and agent-based data across psychological and demographic profiles.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.06440","pdf":"https://arxiv.org/pdf/2607.06440","project":"https://wuyuhang05.github.io/PIPBench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.06440"},"evidence":{"snippet":"To this end, we introduce PIPBench, the first profile-inclusive benchmark for evaluating personalized image generation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06440"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"PIPBench evaluates personalized image generation, where models must align outputs with a user's implicit visual preferences based on a few historically preferred images and a short prompt. It includes real-user and agent-based data across psychological and demographic profiles.","whyItMatters":"Existing text-to-image benchmarks focus on prompt following but ignore individual aesthetic preferences. PIPBench addresses the evaluation gap for personalized generation, offering a standardized way to compare methods aligning outputs with user profiles.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7c3605029971c551431e90afe5d31d46d04c2f5a5e1b1d5c900b143416d8891a"},"motivation":"Recent text-to-image models such as DALLE-3 excel at following diverse prompts yet remain blind to individual aesthetic preferences.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06440","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"PIPBench Team","organizationType":"academic-lab","sourceUrl":"https://wuyuhang05.github.io/PIPBench/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_itpeval_0f85127f","familyId":"bmf_edcd96daf17a","name":"ITPEval","oneLine":"ITPEval evaluates automated formal proof translation across four interactive theorem provers (Lean 4, Rocq, Isabelle, HOL Light), with 1,560 source files and 6,848 theorems, covering statement and proof translation on 12 directed pairs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19407","pdf":"https://arxiv.org/pdf/2607.19407","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19407"},"evidence":{"snippet":"We present ITPEval, the first benchmark for evaluating automated formal proof translation across four major ITPs (Lean 4, Rocq, Isabelle, and HOL Light), spanning two distinct logical foundations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19407"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ITPEval evaluates automated formal proof translation across four interactive theorem provers (Lean 4, Rocq, Isabelle, HOL Light), with 1,560 source files and 6,848 theorems, covering statement and proof translation on 12 directed pairs.","whyItMatters":"ITPEval addresses the lack of a unified benchmark for cross-prover formal proof translation, providing a standardized evaluation of a key capability for automated reasoning and verified software portability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d50cedcc907e1b100a284c56cc9839dbb3cbb7076415c193f377570d249016d1"},"motivation":"Formal theorem proving has emerged as a frontier challenge for machine learning, yet the ecosystem is fragmented: proofs remain siloed across incompatible systems, limiting both training data for learning-based provers and the portability of verified results.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19407","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ITPEval team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.19407","role":"benchmark-publisher"}],"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_holocount_b6a34c73","familyId":"bmf_bd0b3a3f9aa2","name":"HoloCount","oneLine":"HoloCount evaluates multimodal large language models on visual counting tasks across three levels: semantic counting (atomic and property-based enumeration), analytical counting (logical composition via spatial and set-based reasoning), and robustness testing (adverse scenarios and grounded counter-priors). The benchmark uses a hierarchical taxonomy and provides a dataset for evaluation.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.06420","pdf":"https://arxiv.org/pdf/2607.06420","project":"https://mm-mvr.github.io/HoloCount/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.06420"},"evidence":{"snippet":"To address these limitations, we introduce HoloCount, a holistic and diagnostically rich benchmark structured around a three-level hierarchical taxonomy.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06420"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"HoloCount evaluates multimodal large language models on visual counting tasks across three levels: semantic counting (atomic and property-based enumeration), analytical counting (logical composition via spatial and set-based reasoning), and robustness testing (adverse scenarios and grounded counter-priors). The benchmark uses a hierarchical taxonomy and provides a dataset for evaluation.","whyItMatters":"Existing counting benchmarks fail to capture complex failure modes under logical constraints or adversarial conditions. HoloCount provides a structured diagnostic tool to assess MLLM quantitative precision, revealing performance gaps as tasks shift from perception to analytical reasoning, and guiding development of more grounded multimodal systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8a5aecf80c7de9ed617c8b644b55b7fd52e8c32f92a01db2e358afaad9661c72"},"motivation":"Visual counting is a fundamental pillar of multimodal intelligence, requiring a seamless integration of fine-grained grounding and spatial reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06420","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HoloCount Team","organizationType":"academic-lab","sourceUrl":"https://mm-mvr.github.io/HoloCount/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_datagovbench_274a71be","familyId":"bmf_07992656535e","name":"DataGovBench","oneLine":"DataGovBench evaluates LLMs on real-world data analysis using government open data. It comprises Table QA (complex decomposable questions with textual or visual answers) and Table Insight (exploratory data analysis producing expert-level findings). Scoring uses unambiguous reference-based metrics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.06482","pdf":"https://arxiv.org/pdf/2607.06482","project":null,"code":"https://github.com/SoHasegawa/datagovbench","data":null,"hfPaper":"https://huggingface.co/papers/2607.06482"},"evidence":{"snippet":"We introduce DataGovBench, a benchmark derived from governmental open data designed to evaluate LLMs in practical scenarios.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06482"},"ranking":{"90d":{"score":25,"rank":389,"coverage":0.7,"confidence":"Medium"}},"description":"DataGovBench evaluates LLMs on real-world data analysis using government open data. It comprises Table QA (complex decomposable questions with textual or visual answers) and Table Insight (exploratory data analysis producing expert-level findings). Scoring uses unambiguous reference-based metrics.","whyItMatters":"Existing LLM benchmarks miss real-world data complexities like large multi-table datasets and exploratory insight discovery. DataGovBench provides a challenging evaluation to gauge practical readiness of LLMs for data analytics tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"369cde07352dd802105b52d94e627fa762c814a7637f875487176abc53e9914b"},"motivation":"Current benchmarks for evaluating Large Language Models (LLMs) in data analysis often fail to reflect real-world settings.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06482","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DataGovBench Team","organizationType":"academic-lab","sourceUrl":"https://github.com/SoHasegawa/datagovbench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_agentlens_07b9cd48","familyId":"bmf_95050afca81d","name":"AgentLens","oneLine":"AgentLens evaluates interactive coding agents across entire trajectories, combining formal verification with LLM-written trajectory reviews and side-by-side comparisons to score dimensions such as instruction compliance, tool use, and interaction style.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.06624","pdf":"https://arxiv.org/pdf/2607.06624","project":null,"code":"https://github.com/agent-lens/agent-lens-bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.06624"},"evidence":{"snippet":"We present AgentLens, a production-assessed benchmark for interactive code agents.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":9,"hfDailySubmittedAt":null,"githubStars":7,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.06624"},"ranking":{"90d":{"score":58,"rank":144,"coverage":0.7,"confidence":"Medium"}},"description":"AgentLens evaluates interactive coding agents across entire trajectories, combining formal verification with LLM-written trajectory reviews and side-by-side comparisons to score dimensions such as instruction compliance, tool use, and interaction style.","whyItMatters":"Most code-agent benchmarks reduce an episode to a single pass/fail bit, which is too coarse for production use. AgentLens provides a richer evaluation that captures real user experience and yields readable justifications for scores, aiding regression detection and model diagnosis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8c31964b90f1f6e7fe19a4da453e193bd6736a323c42fc0e7a19f051516cc381"},"motivation":"We present AgentLens, a production-assessed benchmark for interactive code agents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.06624","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_whisperbench_c407b81a","familyId":"bmf_253cb261e509","name":"WhisperBench","oneLine":"WhisperBench evaluates stealth memory injection attacks on persistent personal agents through a 108-case benchmark spanning five risk categories with fact and preference poisoning, using an IMAP/SMTP workflow and an email agent skill.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.05189","pdf":"https://arxiv.org/pdf/2607.05189","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.05189"},"evidence":{"snippet":"We introduce WhisperBench, a 108-case benchmark spanning five risk categories and both fact and preference poisoning.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.05189"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"WhisperBench evaluates stealth memory injection attacks on persistent personal agents through a 108-case benchmark spanning five risk categories with fact and preference poisoning, using an IMAP/SMTP workflow and an email agent skill.","whyItMatters":"The benchmark addresses the evaluation gap in assessing security of persistent memory in AI agents, providing a way to measure susceptibility to memory injection attacks and the effectiveness of defenses.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9c6e2c71dfc3405506af7498227236229562bc3cde7c9aa07c021e261ee685f9"},"motivation":"Persistent personal agents combine long-term memory with access to users' external environments, enabling personalized foreground assistance and proactive background execution.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.05189","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_toolfailbench_999a8910","familyId":"bmf_432789540d43","name":"ToolFailBench","oneLine":"ToolFailBench evaluates tool-use failures in LLM agents across 1,000 tasks in finance, medicine, law, cybersecurity, and real estate. It labels traces with Tool-Skip, Result-Ignore, Output-Fabrication, and Unnecessary-Tool-Use, using a rule classifier and LLM judges.","area":"Language & Knowledge","applicationDomains":["Cybersecurity","Finance & Economics"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity","Financial Services"],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.04686","pdf":"https://arxiv.org/pdf/2607.04686","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.04686"},"evidence":{"snippet":"We introduce ToolFailBench, a diagnostic benchmark for measuring tool-use failures across 1,000 tasks in finance, medicine, law, cybersecurity, and real estate.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.04686"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ToolFailBench evaluates tool-use failures in LLM agents across 1,000 tasks in finance, medicine, law, cybersecurity, and real estate. It labels traces with Tool-Skip, Result-Ignore, Output-Fabrication, and Unnecessary-Tool-Use, using a rule classifier and LLM judges.","whyItMatters":"Aggregate accuracy hides distinct failure modes in tool use. This benchmark separates models that fail to call tools from those that call but ignore results, enabling targeted diagnosis of agent reliability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7bd9bd8aa8b7a1e612f0620acdbd7999fe753d02c91af512f02ca6a92f9c527f"},"motivation":"Tool calling is central to modern language model agents, but aggregate benchmark scores often hide where tool use fails.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.04686","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"cross-domain"},{"id":"bm_steelbench_6bf9f31b","familyId":"bmf_c0bc763f6367","name":"SteelBench","oneLine":"STEELBENCH evaluates vision-language models on per-worker activity recognition and safety-rule reasoning in industrial CCTV footage. It includes 1,345 clips with dense annotations and a provenance-aware audit protocol.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Safety","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.05264","pdf":"https://arxiv.org/pdf/2607.05264","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.05264"},"evidence":{"snippet":"We introduce STEELBENCH, a diagnostic benchmark for industrial surveillance that jointly evaluates per-worker activity recognition, safety-rule reasoning, and annotation provenance.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.05264"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"STEELBENCH evaluates vision-language models on per-worker activity recognition and safety-rule reasoning in industrial CCTV footage. It includes 1,345 clips with dense annotations and a provenance-aware audit protocol.","whyItMatters":"Industrial surveillance poses unique visual and procedural challenges. This benchmark reveals large gaps in VLM performance and shows how annotation provenance can inflate accuracy, guiding reliable deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b95fe3291dce8586dd5090da725478438b20b1e36996f31f3e4c6369a81b8646"},"motivation":"Existing video benchmarks evaluate action recognition on consumer videos, egocentric recordings, or simulated industrial environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.05264","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_spearbench_7a039de0","familyId":"bmf_e27542ecd67d","name":"SPEARBench","oneLine":"SPEARBench evaluates naturalness in streaming speech-to-speech language models via question-answer interactions, measuring latency, interruptions, speech quality, ASR robustness, language consistency, emotional naturalness, and interpersonal stance.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.05365","pdf":"https://arxiv.org/pdf/2607.05365","project":"https://thomasthebaud.github.io/SPEAR-benchmark-website/#welcome","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.05365"},"evidence":{"snippet":"We introduce SPEARBench, a benchmark for evaluating naturalness in speech-to-speech language models from question-answer interactions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.05365"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SPEARBench evaluates naturalness in streaming speech-to-speech language models via question-answer interactions, measuring latency, interruptions, speech quality, ASR robustness, language consistency, emotional naturalness, and interpersonal stance.","whyItMatters":"Standard speech metrics miss conversational naturalness. This benchmark provides a multidimensional protocol to assess human-like behavior in spoken interactions, critical for user acceptance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fe69f7bf05f0f643db34786aa8abe7971a6dd7f342d4f0dafcde617207ad320f"},"motivation":"Streaming speech-to-speech language models aim to answer spoken queries directly with synthetic speech.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.05365","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_sovereignpa-bench_ec65a032","familyId":"bmf_dd949d2b2f94","name":"SovereignPA-Bench","oneLine":"SovereignPA-Bench evaluates user-owned personal agents on 120 sovereignty stress scenarios, measuring task success, alignment, privacy, consent, evidence, manipulation, burden, and auditability in evolving intent and platform mediation contexts.","area":"Language & Knowledge","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.05363","pdf":"https://arxiv.org/pdf/2607.05363","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.05363"},"evidence":{"snippet":"We introduce SovereignPA-Bench, an executable benchmark for evaluating user-owned personal agents under evolving intent, platform mediation, privacy boundaries, consent constraints, evidence requirements, and burden tradeoffs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.05363"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SovereignPA-Bench evaluates user-owned personal agents on 120 sovereignty stress scenarios, measuring task success, alignment, privacy, consent, evidence, manipulation, burden, and auditability in evolving intent and platform mediation contexts.","whyItMatters":"Personal agents must balance task completion with user sovereignty. This benchmark quantifies trade-offs across privacy, consent, and manipulation, enabling principled agent design.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d779f2c46df511ec2e6c89eb090de98c1e40fd1096ba1a6774c6ab14a6bb7b3d"},"motivation":"Personal agents are becoming persistent user-owned intermediaries: they remember preferences, filter platform-mediated information, use tools, and negotiate with services.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.05363","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_robovista_4b4e3ac7","familyId":"bmf_3df0c5f42047","name":"RoboVista","oneLine":"RoboVista evaluates vision-language models on robot question answering with 474 visual question answering instances spanning 39 task types across agricultural, industrial, domestic, surgical, and autonomous driving domains.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.04610","pdf":"https://arxiv.org/pdf/2607.04610","project":"https://berkeleyautomation.github.io/robovista/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.04610"},"evidence":{"snippet":"We propose Robot Question Answering (RQA), a modular evaluation framework and RoboVista, a benchmark curated from real robotic systems, research papers, and expert annotations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.04610"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RoboVista evaluates vision-language models on robot question answering with 474 visual question answering instances spanning 39 task types across agricultural, industrial, domestic, surgical, and autonomous driving domains.","whyItMatters":"Robot applications require modular reasoning across diverse embodiments. This benchmark isolates decision components and shows correlation with real-world task execution, aiding VLM selection.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"309dd1beb54804291eca7401b61f4e0b37428c9ffacb78a60113abaf55e93255"},"motivation":"Diverse applications for robotics, such as industry and agriculture, require robots to operate across various embodiments, changing visual conditions, and complex planning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"RSS 2026","evidence":"Accepted to RSS 2026. Project website: https://berkeleyautomation.github.io/robovista/","evidenceUrl":"https://arxiv.org/abs/2607.04610","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"RSS 2026","reviewStatus":"accepted","decisionRaw":"Accepted to RSS 2026. Project website: https://berkeleyautomation.github.io/robovista/","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.04610","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to RSS 2026. Project website: https://berkeleyautomation.github.io/robovista/","level":"author-claim"}]}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_mteb-br_cc2fe8a1","familyId":"bmf_16304415cf38","name":"MTEB-BR","oneLine":"MTEB-BR is a text embedding benchmark for Brazilian Portuguese with 22 native tasks across seven categories, including classification, clustering, retrieval, and reranking. It evaluates models on native Portuguese data and provides a public leaderboard.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.04581","pdf":"https://arxiv.org/pdf/2607.04581","project":"https://doi.org/10.5281/zenodo.21087216","code":"https://github.com/tardellirs/mteb-br","data":null,"hfPaper":"https://huggingface.co/papers/2607.04581"},"evidence":{"snippet":"We introduce MTEB-BR, a benchmark of 22 native Brazilian-Portuguese tasks across seven categories (classification, multilabel classification, pair classification, semantic textual similarity, clustering, retrieval, and reranking), admitting only data created or found in Portuguese and excluding translations by construction.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":10,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.04581"},"ranking":{"90d":{"score":54,"rank":172,"coverage":0.7,"confidence":"Medium"}},"description":"MTEB-BR is a text embedding benchmark for Brazilian Portuguese with 22 native tasks across seven categories, including classification, clustering, retrieval, and reranking. It evaluates models on native Portuguese data and provides a public leaderboard.","whyItMatters":"Portuguese embedding evaluation lacked native benchmarks. This benchmark provides statistically grounded model comparison, showing that multilingual leaderboards only moderately predict Portuguese performance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1c20f055685d32699e3ebefb3a09fa498ff4594988aebef3557c3af32a27ccb8"},"motivation":"Text embeddings for Portuguese have no dedicated benchmark: evaluation rests on translated corpora such as English MS MARCO or on thin multilingual coverage, with native tasks scattered and unconsolidated.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.04581","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_hamqasbench_9db953d2","familyId":"bmf_9eb34c8bc18c","name":"HamQASBench","oneLine":"HamQASBench evaluates Quantum Architecture Search methods across 11 molecules organized in five structural tiers, using energy accuracy, per-qubit entanglement, and pairwise state fidelity to diagnose structural failures.","area":"Language & Knowledge","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Quantum Technology"],"capabilities":[],"topics":["quant-ph"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.04845","pdf":"https://arxiv.org/pdf/2607.04845","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.04845"},"evidence":{"snippet":"We introduce HamQASBench, a Hamiltonian-informed diagnostic benchmark organizing 11 molecules into five structural tiers via fingerprints derived from the Pauli operator basis, computational basis representation, and ground-state entanglement.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.04845"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HamQASBench evaluates Quantum Architecture Search methods across 11 molecules organized in five structural tiers, using energy accuracy, per-qubit entanglement, and pairwise state fidelity to diagnose structural failures.","whyItMatters":"Existing QAS benchmarks miss structural failures like over-parameterization. This benchmark provides diagnostic metrics that reveal method-specific weaknesses, guiding algorithm development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"98fa095813bd8c2a2ba957fc87d0003925ab6a8e53ac13e1fcec04b21ae308df"},"motivation":"Quantum Architecture Search (QAS) automates the design of parameterized quantum circuits for variational quantum algorithms, yet existing benchmarks organize instances by molecular identity or qubit count -- criteria agnostic to Hamiltonian structure -- and rely solely on energy accuracy, which cannot detect structural failures such as over-parameterization on near-product ground states.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.04845","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_evoagentbench_cbe93d38","familyId":"bmf_3e6250cd3257","name":"EvoAgentBench","oneLine":"EvoAgentBench evaluates agent self-evolution via ability transfer across web research, algorithmic reasoning, software engineering, and knowledge work, using trace-grounded abilities and domain graphs with train/test splits.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Reasoning","Factuality"],"topics":["Self-Evolution","Agents","Code"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Inspectable","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.05202","pdf":"https://arxiv.org/pdf/2607.05202","project":null,"code":null,"data":"https://huggingface.co/datasets/EverMind-AI/EvoAgentBench","hfPaper":"https://huggingface.co/papers/2607.05202"},"evidence":{"snippet":"We introduce EvoAgentBench, a benchmark for agent self-evolution via Ability-guided transfer across four agentic domains: web research, algorithmic reasoning, software engineering, and knowledge work.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":323,"hfDatasetLikes":16},"source":{"type":"arxiv","id":"2607.05202"},"ranking":{"90d":{"score":55,"rank":166,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":17,"datasetRankPopulation":51}},"description":"EvoAgentBench evaluates agent self-evolution via ability transfer across web research, algorithmic reasoning, software engineering, and knowledge work, using trace-grounded abilities and domain graphs with train/test splits.","whyItMatters":"Current evaluations don't isolate procedural reuse. This benchmark enables fine-grained diagnosis of experience encoding, routing, and uptake in agent self-evolution.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c159f16a4c3d63f272f17bea01e82196c6399f357ea0c0e5bd90a169b8e170f7"},"motivation":"Agent self-evolution in long-horizon LLM systems is largely procedural: useful experience is not merely stored information, but reusable procedures for searching, debugging, and verification.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.05202","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_edgebench_88dc3c92","familyId":"bmf_2c4d59d4e34e","name":"EdgeBench","oneLine":"EdgeBench evaluates autonomous agents on 134 real-world tasks across scientific discovery, software engineering, optimization, knowledge work, formal mathematics, and games. Each task requires 12+ hours of continuous interaction with multi-level feedback. Scoring tracks agent performance over time (at 2,4,6,8,10,12 hours). A public leaderboard is maintained; 51 tasks and evaluation framework are open-sourced.","area":"Agents & Tool Use","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":[],"capabilities":["Environment learning","Long-horizon task execution","Learning from feedback"],"topics":["Agents","Environment Learning","Scaling Laws"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.05155","pdf":"https://arxiv.org/pdf/2607.05155","project":"https://edge-bench.org/","code":"https://github.com/ByteDance-Seed/EdgeBench","data":"https://huggingface.co/datasets/ByteDance-Seed/EdgeBench","hfPaper":"https://huggingface.co/papers/2607.05155"},"evidence":{"snippet":"This discovery stems from EdgeBench, a suite of 134 real world tasks with ultra-long horizons, spanning scientific discovery, software engineering, combinatorial optimization, professional knowledge work, formal mathematics, and interactive games. […] We publicly release 51 tasks and our full evaluation framework to accelerate the study of how agents learn from real world experience.","reasonCodes":["coined title prefix ending in Bench or Benchmark","named benchmark identity and task release stated in nearby sentences","evaluation protocol evidence"]},"dataStatus":"primary-source-reviewed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":18,"hfDailySubmittedAt":null,"githubStars":431,"githubScope":"benchmark_repo","hfDatasetDownloads":4999,"hfDatasetLikes":83},"source":{"type":"arxiv","id":"2607.05155"},"ranking":{"90d":{"score":95,"rank":1,"coverage":1.0,"confidence":"High","datasetDownloadRank":3,"datasetRankPopulation":51}},"description":"EdgeBench evaluates autonomous agents on 134 real-world tasks across scientific discovery, software engineering, optimization, knowledge work, formal mathematics, and games. Each task requires 12+ hours of continuous interaction with multi-level feedback. Scoring tracks agent performance over time (at 2,4,6,8,10,12 hours). A public leaderboard is maintained; 51 tasks and evaluation framework are open-sourced.","whyItMatters":"EdgeBench fills a gap in evaluating agents' ability to learn from real-world environments over extended periods, providing a standardized protocol for comparing long-horizon learning capabilities. It offers a public leaderboard and open-source tasks, enabling reproducible comparisons and tracking of agent improvement over time, which is valuable for model selection and development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bf947e2af808593cdcd52d4e4578c40a22f3628308d6650745e5a65db4ffa6b3"},"motivation":"Pretraining scaling laws reveal that model capability improves predictably with data and compute.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"source-reviewed","reviewedAt":"2026-08-19","sources":["https://arxiv.org/abs/2607.05155","https://edge-bench.org/","https://github.com/ByteDance-Seed/EdgeBench","https://huggingface.co/datasets/ByteDance-Seed/EdgeBench"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.05155","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"releaseDates":{"firstPublicAt":"2026-07-02","paperV1At":"2026-07-06"},"publishers":[{"name":"ByteDance Seed","organizationType":"company-research-lab","sourceUrl":"https://github.com/ByteDance-Seed/EdgeBench","role":"benchmark-publisher"}],"capabilityGroups":["Agents","Coding & Software Engineering","Mathematics & Formal Sciences"],"domainScope":"specific"},{"id":"bm_cstutorbench_cf78c812","familyId":"bmf_0e6e4bdbb6d8","name":"CSTutorBench","oneLine":"CSTutorBench evaluates small language models as tutors in VEX VR block-based programming, with 17 scenario-based questions scored via a rubric and LLM-as-judge.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Code generation"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.05571","pdf":"https://arxiv.org/pdf/2607.05571","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.05571"},"evidence":{"snippet":"We introduce CSTutorBench, a benchmark for evaluating language models as CS tutors in VEX VR, a block-based robotics environment.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.05571"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CSTutorBench evaluates small language models as tutors in VEX VR block-based programming, with 17 scenario-based questions scored via a rubric and LLM-as-judge.","whyItMatters":"The benchmark addresses the gap in evaluating SLMs for block-based programming tutoring, where models often lack domain-specific training.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d042a9cc0d83045cb479a5562f2ff6a122de902e8ed06dd8373c6364e06c7f70"},"motivation":"Large language models are increasingly explored as AI tutors, yet deploying them in K-12 settings raises concerns around privacy, cost, and reliance on proprietary models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"publication_reported","venue":"SLM4ED'26: The 1st Workshop of Small Language Models for Education (SLM4ED). AIED 2026. Seoul, Republic of Korea","evidence":"SLM4ED'26: The 1st Workshop of Small Language Models for Education (SLM4ED). AIED 2026. Seoul, Republic of Korea","evidenceUrl":"https://arxiv.org/abs/2607.05571","source":"arxiv-journal-reference","evidenceLevel":"strong-author-metadata","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publications":[{"venueName":"SLM4ED'26: The 1st Workshop of Small Language Models for Education (SLM4ED). AIED 2026. Seoul, Republic of Korea","publicationStatus":"published","evidence":[{"sourceType":"arxiv-journal-reference","sourceUrl":"https://arxiv.org/abs/2607.05571","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"SLM4ED'26: The 1st Workshop of Small Language Models for Education (SLM4ED). AIED 2026. Seoul, Republic of Korea","level":"strong-author-metadata"}]}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_classiclogic_43434cd2","familyId":"bmf_9984193cf2db","name":"ClassicLogic","oneLine":"ClassicLogic is a benchmark suite of four classic logic puzzles (Sudoku, KenKen, Kakuro, Futoshiki) with a hierarchical knowledge base that defines complex strategies as compositions of simpler ones. It evaluates an agent's compositional generalization in problem-solving from basic rules to multi-step strategies across increasing puzzle difficulty.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.05185","pdf":"https://arxiv.org/pdf/2607.05185","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.05185"},"evidence":{"snippet":"We introduce ClassicLogic, a new benchmark suite designed to evaluate an agent's ability to learn and compose problem-solving strategies.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.05185"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ClassicLogic is a benchmark suite of four classic logic puzzles (Sudoku, KenKen, Kakuro, Futoshiki) with a hierarchical knowledge base that defines complex strategies as compositions of simpler ones. It evaluates an agent's compositional generalization in problem-solving from basic rules to multi-step strategies across increasing puzzle difficulty.","whyItMatters":"Most compositional generalization benchmarks focus on language; ClassicLogic provides a structured, non-linguistic testbed with explicit compositional logic. It enables fine-grained assessment of reasoning capabilities and supports development of neuro-symbolic systems capable of systematic problem-solving.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fa6e7171734804034fc7bea25e3d4675f1b364ec427822776a6ac2bd3a51dbe1"},"motivation":"Compositional generalization, the ability to understand and produce novel combinations of known components, remains a fundamental challenge for modern artificial intelligence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.05185","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ClassicLogic Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.05185","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_biosecbench-refusal_d798db06","familyId":"bmf_f88a84efd022","name":"BioSecBench-Refusal","oneLine":"BioSecBench-Refusal evaluates AI agents on 61 Routine and 46 Red-Team biosecurity-related tasks, measuring refusal rates and risk identification across multiple model configurations.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.05462","pdf":"https://arxiv.org/pdf/2607.05462","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.05462"},"evidence":{"snippet":"We present BioSecBench-Refusal, a benchmark for risk identification and refusal behavior for biological research tasks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.05462"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"BioSecBench-Refusal evaluates AI agents on 61 Routine and 46 Red-Team biosecurity-related tasks, measuring refusal rates and risk identification across multiple model configurations.","whyItMatters":"Addresses the need for benchmarks that quantify both capability and safety in agentic biosecurity, helping developers calibrate models to avoid over-refusal while still detecting threats.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"71677a53bbce6ee720e8803b2e8c1cf8f83cbb421da9d89c5b1b99e1e5d8bf29"},"motivation":"As AI agents are incorporated into life science workflows, the capabilities that speed discovery might also enable misuse.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.05462","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_air-bench_2ac1e887","familyId":"bmf_93bc2ca98a94","name":"AIR-BENCH","oneLine":"AIR-BENCH Live is a self-evolving safety benchmark for foundation models, with an automated pipeline that updates risk taxonomy and prompts based on new regulations. It evaluates model safety across multilingual prompts.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-06","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.22671","pdf":"https://arxiv.org/pdf/2607.22671","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.22671"},"evidence":{"snippet":"We present AIR-BENCH Live, a self-evolving successor to AIR-BENCH 2024.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.22671"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AIR-BENCH Live is a self-evolving safety benchmark for foundation models, with an automated pipeline that updates risk taxonomy and prompts based on new regulations. It evaluates model safety across multilingual prompts.","whyItMatters":"This benchmark aims to keep pace with evolving AI risks and regulations, providing a dynamic evaluation tool for model safety. Its automated updates could help maintain relevance, but the lack of a fixed protocol limits comparability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4d85cbc5c6e85eed93304ef2d8a7f3d283d110a3a2c7f2150feca83c42363705"},"motivation":"Foundation-model safety benchmarks capture the AI risks of their time of publication: as models improve and governments pass new AI-safety legislation, their risk taxonomies become incomprehensive and their attack prompts become ineffective.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.22671","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_softvtbench_b731832c","familyId":"bmf_eb5e5d2a1a04","name":"SoftVTBench","oneLine":"SoftVTBench is a safety-aware visuo-tactile benchmark for deformable object manipulation in Isaac Sim, with FEM-simulated objects, multi-view RGB, tactile sensing, proprioception, language instructions, and separate Goal Success and Safety Success metrics.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.04234","pdf":"https://arxiv.org/pdf/2607.04234","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.04234"},"evidence":{"snippet":"We present SoftVTBench, a safety-aware visuo-tactile benchmark for physically constrained deformable object manipulation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.04234"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SoftVTBench is a safety-aware visuo-tactile benchmark for deformable object manipulation in Isaac Sim, with FEM-simulated objects, multi-view RGB, tactile sensing, proprioception, language instructions, and separate Goal Success and Safety Success metrics.","whyItMatters":"Existing manipulation benchmarks focus on task success and overlook physical safety, such as avoiding drops or excessive deformation. SoftVTBench addresses this gap by providing a protocol that evaluates both goal achievement and safety, offering decision value for developing safer robotic manipulation policies.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"70ef8c359f39494bdcaf94033c185b8bfae3ec293dab0e09e84d968eac48357c"},"motivation":"Deformable object manipulation poses challenges beyond task completion: successful execution must also maintain safe physical interaction, holding the object stably without slip or drop while avoiding excessive deformation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCVW","evidence":"Early version of SoftVTBench, Accepted by ECCVW","evidenceUrl":"https://arxiv.org/abs/2607.04234","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCVW","reviewStatus":"accepted","decisionRaw":"Early version of SoftVTBench, Accepted by ECCVW","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.04234","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Early version of SoftVTBench, Accepted by ECCVW","level":"author-claim"}]}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_robodojo_1fa2182f","familyId":"bmf_1b4dac6f946e","name":"RoboDojo","oneLine":"RoboDojo is a unified sim-and-real benchmark for evaluating generalist robot manipulation policies, comprising 42 simulation tasks and 18 real-world tasks across three robot embodiments, covering five capability dimensions: generalization, memory, precision, long-horizon execution, and open-vocabulary instruction following.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.04434","pdf":"https://arxiv.org/pdf/2607.04434","project":"http://robodojo-benchmark.com/","code":"https://github.com/RoboDojo-Benchmark/RoboDojo","data":null,"hfPaper":"https://huggingface.co/papers/2607.04434"},"evidence":{"snippet":"We introduce RoboDojo, a unified sim-and-real benchmark for comprehensive evaluation of generalist robot manipulation policies.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":16,"hfDailySubmittedAt":null,"githubStars":421,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.04434"},"ranking":{"90d":{"score":81,"rank":16,"coverage":0.7,"confidence":"Medium"}},"description":"RoboDojo is a unified sim-and-real benchmark for evaluating generalist robot manipulation policies, comprising 42 simulation tasks and 18 real-world tasks across three robot embodiments, covering five capability dimensions: generalization, memory, precision, long-horizon execution, and open-vocabulary instruction following.","whyItMatters":"Existing benchmarks often limit scope to narrow tasks or single settings. RoboDojo provides a comprehensive, reproducible evaluation across simulation and real world, enabling systematic comparison of policies through a public leaderboard and cloud-based real-world evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e24539efe71fb44b9e46c2e09dea791b0d155e38e01b2a86d5552b438dbcf5b5"},"motivation":"Generalist robot manipulation policies have advanced rapidly, yet existing benchmarks remain limited in systematically evaluating their capabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.04434","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"RoboDojo Benchmark Team","organizationType":"academic-lab","sourceUrl":"http://robodojo-benchmark.com/","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_hribench_c65825aa","familyId":"bmf_9b6d4c3d73e5","name":"HRIBench","oneLine":"HRIBench evaluates intent-aware human-robot collaboration through structured scenario scripts covering roles of Instructor, Collaborator, and Intruder, with 13 tasks and 650 episodes, scoring synchronization, responsiveness, protocol compliance, and safety.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.13056","pdf":"https://arxiv.org/pdf/2607.13056","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.13056"},"evidence":{"snippet":"To address this gap, we introduce HRIBench, a diagnostic benchmark for intent-aware human-robot collaboration based on executable interaction scenarios.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.13056"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"HRIBench evaluates intent-aware human-robot collaboration through structured scenario scripts covering roles of Instructor, Collaborator, and Intruder, with 13 tasks and 650 episodes, scoring synchronization, responsiveness, protocol compliance, and safety.","whyItMatters":"Existing VLA benchmarks focus on isolated manipulation, leaving a gap in evaluating coordination under shared agency. HRIBench provides a standardized protocol for assessing temporal coordination and intent understanding, with evidence that fine-tuning on its data improves real-world task success.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7e1e7e0b42abdc53a3334f4312f2089ded4d9900e0e8dc77aa433188ecd3dd9d"},"motivation":"Current vision-language-action (VLA) benchmarks primarily evaluate isolated manipulation skills while leaving human-robot interaction structure largely unmodeled.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.13056","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HRIBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.13056","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_freshcache-bench_dd554470","familyId":"bmf_e44b422adf38","name":"FreshCache-Bench","oneLine":"FreshCache-Bench provides 8,072 base queries across five freshness classes with ground truth staleness labels from web snapshots at 1, 12, 24 hours, and 7 days, expanded to 31,201 queries via paraphrase generation, for evaluating semantic caching in retrieval-augmented LLMs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.04281","pdf":"https://arxiv.org/pdf/2607.04281","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.04281"},"evidence":{"snippet":"We introduce FreshCache-Bench, a benchmark of 8,072 base queries across five freshness classes with ground truth staleness labels drawn from real web snapshots at 1, 12, 24 hours, and 7 days after a baseline crawl, expanded to 31,201 queries via paraphrase generation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.04281"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FreshCache-Bench provides 8,072 base queries across five freshness classes with ground truth staleness labels from web snapshots at 1, 12, 24 hours, and 7 days, expanded to 31,201 queries via paraphrase generation, for evaluating semantic caching in retrieval-augmented LLMs.","whyItMatters":"Semantic caching for RAG lacks standardized evaluation of freshness; this benchmark addresses the gap by quantifying stale error and search API savings, enabling comparison of caching strategies in terms of cost and correctness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"682cefc0af17c68f6426c0ac6c4d344b649fe55755c77570df580281e3fa2b5d"},"motivation":"Semantic caching reduces the latency and cost of retrieval-augmented generation (RAG) by serving cached answers to semantically similar queries, but most existing methods do not model the time-varying freshness of open-web evidence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.04281","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_causalgame_db1d5cd5","familyId":"bmf_533dd8b8578b","name":"CausalGame","oneLine":"CausalGame is a benchmark for evaluating causal thinking in LLM agents through interactive games. It includes 14 scenarios with selection bias, measurement error, and hidden confounders. Agents design experimental protocols, collect data, and produce solutions with explanations, scored against analytical optima and causal-reasoning rubrics.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.04293","pdf":"https://arxiv.org/pdf/2607.04293","project":"https://causalgame.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.04293"},"evidence":{"snippet":"To this end, we present CausalGame, a benchmark that evaluates the causal thinking capabilities of LLM agents through interactive games.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.04293"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CausalGame is a benchmark for evaluating causal thinking in LLM agents through interactive games. It includes 14 scenarios with selection bias, measurement error, and hidden confounders. Agents design experimental protocols, collect data, and produce solutions with explanations, scored against analytical optima and causal-reasoning rubrics.","whyItMatters":"Existing AI Scientist benchmarks do not isolate causal reasoning under realistic biases. CausalGame provides a structured evaluation for this capability, offering decision value for developers of autonomous research agents seeking to assess robustness against confounders and biases.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9c8fa6ded882bcecb48d786209c9442b6ea865d7ac763f993e6a4de6b33a4acb"},"motivation":"Building AI Scientist agents with Large Language Models (LLMs) has recently attracted growing attention.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"Forty-Third International Conference on Machine Learning (ICML) 2026 as an Oral presentation","evidence":"Zhenhao, Yongqiang, and Chenxi contributed equally to the project. A short version is accepted at the Forty-Third International Conference on Machine Learning (ICML) 2026 as an Oral presentation. Project website https://causalgame.github.io/","evidenceUrl":"https://arxiv.org/abs/2607.04293","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"Forty-Third International Conference on Machine Learning (ICML) 2026 as an Oral presentation","reviewStatus":"accepted","decisionRaw":"Zhenhao, Yongqiang, and Chenxi contributed equally to the project. A short version is accepted at the Forty-Third International Conference on Machine Learning (ICML) 2026 as an Oral presentation. Project website https://causalgame.github.io/","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.04293","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Zhenhao, Yongqiang, and Chenxi contributed equally to the project. A short version is accepted at the Forty-Third International Conference on Machine Learning (ICML) 2026 as an Oral presentation. Project website https://causalgame.github.io/","level":"author-claim"}]}],"publishers":[{"name":"CausalGame Team","organizationType":"academic-lab","sourceUrl":"https://causalgame.github.io/","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_tabquerybench_d985f8e2","familyId":"bmf_bdd17825921f","name":"TabQueryBench","oneLine":"TabQueryBench evaluates synthetic tabular data generators using SQL-shaped analytical queries as structural assessors, providing 44 reusable query templates across 49 datasets.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.DB"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-04","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.03926","pdf":"https://arxiv.org/pdf/2607.03926","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.03926"},"evidence":{"snippet":"We present TabQueryBench, a query-centric benchmark that uses SQL-shaped analytical queries as structural assessors for synthetic data fidelity.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.03926"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TabQueryBench evaluates synthetic tabular data generators using SQL-shaped analytical queries as structural assessors, providing 44 reusable query templates across 49 datasets.","whyItMatters":"Existing synthetic data evaluations focus on statistical similarity and downstream ML utility, but rarely test analytical query structure. This benchmark addresses that gap, offering a way to assess query-centric fidelity for practical data analysis use cases.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"980fc5b0a5c4dc58654070e0131a5581188e0bc525f962e85db4c94197ef6778"},"motivation":"Synthetic tabular data support use cases like data sharing, model development under access restrictions, and rapid prototyping of analytical workflows.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.03926","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_omniopt_f648d471","familyId":"bmf_15c40ef3c174","name":"OmniOpt","oneLine":"A cross-domain benchmark for comparing optimizers in large-scale model training. It covers 24+ optimizers across two stages: Stage 1 sweeps on C4 with LLaMA-3 architectures (60M to 1B), and Stage 2 transfers to FineWeb-Edu with four architectures (Transformer++, GLA, DeltaNet, Gated DeltaNet) at 340M and 1B scales. Controlled-variable protocol with fixed architecture, data, and schedule settings.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-04","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.04033","pdf":"https://arxiv.org/pdf/2607.04033","project":null,"code":"https://github.com/OpenRaiser/OmniOpt","data":null,"hfPaper":"https://huggingface.co/papers/2607.04033"},"evidence":{"snippet":"We therefore present OmniOpt, a unified survey and benchmark cookbook of optimizers for the research community.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":76,"hfDailySubmittedAt":null,"githubStars":39,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.04033"},"ranking":{"90d":{"score":76,"rank":43,"coverage":0.7,"confidence":"Medium"}},"description":"A cross-domain benchmark for comparing optimizers in large-scale model training. It covers 24+ optimizers across two stages: Stage 1 sweeps on C4 with LLaMA-3 architectures (60M to 1B), and Stage 2 transfers to FineWeb-Edu with four architectures (Transformer++, GLA, DeltaNet, Gated DeltaNet) at 340M and 1B scales. Controlled-variable protocol with fixed architecture, data, and schedule settings.","whyItMatters":"Optimizer selection is a system-level decision impacting compute, memory, and tuning budget. This benchmark provides a unified protocol for comparing methods across multiple scales and architectures, offering reproducible evidence for practitioners to choose optimizers based on measured training objectives and trade-offs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"297971f39f8cba69d19bc299168f8baf176f30baca420701c21f92ebd23f4510"},"motivation":"Optimizer selection for large-scale model training has become a system-level design decision constrained jointly by compute, memory, tuning budget, and task diversity, yet the landscape of over one hundred methods remains fragmented.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.04033","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"OmniOpt Team","organizationType":"community","sourceUrl":"https://github.com/OpenRaiser/OmniOpt","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_lh-avln_cb4be7f0","familyId":"bmf_bd018386fef4","name":"LH-AVLN","oneLine":"LH-AVLN is a benchmark for long-horizon navigation that combines multi-goal missions, heterogeneous goal specifications, and persistent spatialized audio cues. Agents must execute missions of two to four goals specified by category, language, or reference image, using RGB-D, pose, and binaural audio in indoor 3D environments. It supports ordered and unordered missions with alternating goal-associated sounds.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.RO"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.03920","pdf":"https://arxiv.org/pdf/2607.03920","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.03920"},"evidence":{"snippet":"We introduce LH-AVLN, a benchmark for Long-Horizon Audio-Visual-Language Navigation that combines multi-goal mission execution, heterogeneous goal specifications, and persistent spatialized acoustic cues.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.03920"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LH-AVLN is a benchmark for long-horizon navigation that combines multi-goal missions, heterogeneous goal specifications, and persistent spatialized audio cues. Agents must execute missions of two to four goals specified by category, language, or reference image, using RGB-D, pose, and binaural audio in indoor 3D environments. It supports ordered and unordered missions with alternating goal-associated sounds.","whyItMatters":"Existing navigation benchmarks do not integrate long-horizon missions with audio-visual cues and heterogeneous goal types, while audio-visual tasks are typically single-goal. LH-AVLN addresses this gap by providing a multi-goal environment with acoustic guidance and distractors, enabling more realistic evaluation of embodied agents in complex missions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"980a8dd752a8b2cef97ca30f6aa3f5b1eeb18796d059e953b0118af450b16ed1"},"motivation":"Embodied navigation is moving toward long-horizon missions, yet existing long-horizon benchmarks are largely acoustically silent, and audio-visual navigation tasks typically focus on a single goal.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.03920","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_clinocr-bench_8be5c7a7","familyId":"bmf_6c9fc7398ebb","name":"ClinOCR-Bench","oneLine":"ClinOCR-Bench is a public dataset of 384 scanned clinical documents across six artifact subsets (normal, handwriting, poor quality, rotation, tables, mixed). It evaluates OCR systems on clinical text extraction, with ground truth transcriptions and template-aware train/test splits supporting 0-shot and 1-shot evaluation.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.03650","pdf":"https://arxiv.org/pdf/2607.03650","project":null,"code":"https://github.com/ClinOCR-Bench/ClinOCR-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.03650"},"evidence":{"snippet":"Therefore, we release a publicly available, realistic-looking OCR benchmark dataset, ClinOCR-Bench, with 384 scanned images across 6 subsets: Normal, Handwriting, Poor Quality, Rotation, Tables, and Mix-artifacts.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.03650"},"ranking":{"90d":{"score":44,"rank":246,"coverage":0.55,"confidence":"Low"}},"description":"ClinOCR-Bench is a public dataset of 384 scanned clinical documents across six artifact subsets (normal, handwriting, poor quality, rotation, tables, mixed). It evaluates OCR systems on clinical text extraction, with ground truth transcriptions and template-aware train/test splits supporting 0-shot and 1-shot evaluation.","whyItMatters":"Existing clinical OCR evaluations rely on private data lacking common scan artifacts. ClinOCR-Bench provides a standardized, realistic set of clinical documents with controlled artifacts, enabling reproducible comparison of OCR and vision-language models on real-world clinical scanning challenges.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7fa6bc0c3ec964067f111d5cdd8036b541d6ebdf4c991b8b6b834230ce37a648"},"motivation":"Extracting textual information from scanned medical documents, such as external laboratory reports and manually filled forms, has been a major challenge in modern electronic health records (EHRs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.03650","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_asob-bench_b6e17903","familyId":"bmf_5360c6d29e43","name":"ASOB-Bench","oneLine":"ASOB-Bench evaluates diffusion classifiers along three bias dimensions: attribute binding, size-order bias, and background dependency, using newly constructed datasets and extending existing frameworks.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.03831","pdf":"https://arxiv.org/pdf/2607.03831","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.03831"},"evidence":{"snippet":"We introduce ASOB-Bench, a bias evaluation for diffusion classifiers along three dimensions: Attribute binding, Size-Order bias, and Background dependency.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.03831"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ASOB-Bench evaluates diffusion classifiers along three bias dimensions: attribute binding, size-order bias, and background dependency, using newly constructed datasets and extending existing frameworks.","whyItMatters":"This probe reveals distinct bias profiles in diffusion classifiers compared to vision-language models, informing robustness improvements in diffusion-based systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ef93e04d95b5c0a37ea91b183df23af0236be39ceb60d8c47fe800405fe2705f"},"motivation":"Diffusion models have recently been repurposed for zero-shot classification, giving rise to diffusion classifiers that identify the best-matching text prompt by minimizing the noise-prediction error.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.03831","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_xplainverse_db4c59ee","familyId":"bmf_b434c8c362ed","name":"XPlainVerse","oneLine":"XPlainVerse evaluates deepfake detection and explanation quality, pairing real images with forgeries from twelve models and providing technical and simplified explanations, with metrics EntityScore and EvidenceScore for reasoning fidelity.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.03562","pdf":"https://arxiv.org/pdf/2607.03562","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.03562"},"evidence":{"snippet":"To this end, we introduce XPlainVerse, a large-scale benchmark designed for joint deepfake detection and human-centered explanation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.03562"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"XPlainVerse evaluates deepfake detection and explanation quality, pairing real images with forgeries from twelve models and providing technical and simplified explanations, with metrics EntityScore and EvidenceScore for reasoning fidelity.","whyItMatters":"Existing benchmarks focus on classification accuracy, not explanation grounding. XPlainVerse aims to measure whether explanations are grounded in actual manipulations, which is key for trustworthy deployable detection.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f81da848a9b1ff7c7f80672b7085e5f505787ca6dd37da08bd6d297db90d1e10"},"motivation":"As deepfake detection models increasingly produce natural language explanations, their reasoning often remains weakly grounded in visual artifacts, limiting reliability and user trust.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.03562","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ppe-bench_ab4b6b83","familyId":"bmf_74f307632439","name":"PPE-Bench","oneLine":"PPE-Bench evaluates machine unlearning in multimodal large language models under private-public entanglement, where images contain a target individual to forget and public elements to preserve.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.02897","pdf":"https://arxiv.org/pdf/2607.02897","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.02897"},"evidence":{"snippet":"To address these limitations, we propose PPE-Bench, a new benchmark for evaluating MLLM unlearning under private-public entanglement.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02897"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PPE-Bench evaluates machine unlearning in multimodal large language models under private-public entanglement, where images contain a target individual to forget and public elements to preserve.","whyItMatters":"Addresses the lack of benchmarks that reflect real-world image complexity and entanglement of private and public information, supporting evaluation of unlearning methods that must preserve public context.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0b6b09e58576b7f26989245ac1e30cc6b20463c1104f41034d141ce4992e1797"},"motivation":"Multimodal Large Language Models (MLLMs) have shown strong capabilities, but they may memorize private information from web data, raising privacy concerns.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02897","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_omnilayout_9c41426f","familyId":"bmf_c7f5b7944f24","name":"OmniLayout","oneLine":"OmniLayout evaluates language models on printed-circuit-board (PCB) layout placement reasoning under geometric, routing, and connectivity constraints, with four tasks including geometric placement, routability-aware placement, electrical functionality, and tool-augmented agentic reasoning.","area":"Multimodal","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Semiconductors"],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.03261","pdf":"https://arxiv.org/pdf/2607.03261","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.03261"},"evidence":{"snippet":"To bridge this gap, we introduce OmniLayout, the first benchmark designed to evaluate LLMs on printed-circuit-board (PCB) layout placement reasoning under real-world geometric, routing, and connectivity constraints.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.03261"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OmniLayout evaluates language models on printed-circuit-board (PCB) layout placement reasoning under geometric, routing, and connectivity constraints, with four tasks including geometric placement, routability-aware placement, electrical functionality, and tool-augmented agentic reasoning.","whyItMatters":"The benchmark addresses the gap in evaluating LLMs for practical electronic design automation (EDA) tasks, specifically constraint-aware geometric reasoning in PCB layout, which is critical for real-world design workflows.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ff1c5ccad8c9f7a650a1da75a0fce42ea1c4a454c126e1b59937a4278db41434"},"motivation":"Recent large language models (LLMs) have demonstrated remarkable progress in 3D spatial reasoning, spatial grounding, and fine-grained geometric understanding.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.03261","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_more_09a372c7","familyId":"bmf_187897ce0afc","name":"MORE","oneLine":"MORE evaluates multilingual document parsing across 149 languages, covering text, formulas, tables, code blocks, catalogs, and reading order, using 1,288 real-world document images with human-refined annotations.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.02956","pdf":"https://arxiv.org/pdf/2607.02956","project":null,"code":"https://github.com/zimoqingfeng/MORE","data":null,"hfPaper":"https://huggingface.co/papers/2607.02956"},"evidence":{"snippet":"To bridge this gap, we introduce MORE, a large-scale benchmark designed for multilingual document parsing evaluation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02956"},"ranking":{"90d":{"score":49,"rank":200,"coverage":0.7,"confidence":"Medium"}},"description":"MORE evaluates multilingual document parsing across 149 languages, covering text, formulas, tables, code blocks, catalogs, and reading order, using 1,288 real-world document images with human-refined annotations.","whyItMatters":"Existing benchmarks focus on high-resource languages, leaving a gap in assessing multilingual document parsers. MORE provides a broad and structured evaluation to compare model capabilities on low-resource languages and diverse document elements.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"de191e91e618d2ef15e6d3be694619cb5aac0ea95ba4b04d80f106157c7c42e5"},"motivation":"Multilingual documents encapsulate rich regional cultures, scientific discoveries, and historical records.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"43rd International Conference on Machine Learning (ICML 2026)","evidence":"Accepted to the 43rd International Conference on Machine Learning (ICML 2026). 22 pages, 11 figures. Code and dataset available at https://github.com/zimoqingfeng/MORE","evidenceUrl":"https://arxiv.org/abs/2607.02956","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"43rd International Conference on Machine Learning (ICML 2026)","reviewStatus":"accepted","decisionRaw":"Accepted to the 43rd International Conference on Machine Learning (ICML 2026). 22 pages, 11 figures. Code and dataset available at https://github.com/zimoqingfeng/MORE","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.02956","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to the 43rd International Conference on Machine Learning (ICML 2026). 22 pages, 11 figures. Code and dataset available at https://github.com/zimoqingfeng/MORE","level":"author-claim"}]}],"publishers":[{"name":"MORE Benchmark Team","organizationType":"community","sourceUrl":"https://github.com/zimoqingfeng/MORE","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_matphasebench_87d9e805","familyId":"bmf_77353b9cc428","name":"MatPhaseBench","oneLine":"MatPhaseBench evaluates vision-language models on understanding materials phase diagrams, using 200 diagram-text pairs from 3681 papers. It targets complex scientific image understanding, with tasks requiring deep comprehension and open-ended responses, covering 189 material systems and 70 elements.","area":"Vision & 3D","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.02934","pdf":"https://arxiv.org/pdf/2607.02934","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.02934"},"evidence":{"snippet":"We introduce MatPhaseBench, a high-quality, high-reliability benchmark for complex scientific image understanding, focused on materials phase diagrams.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02934"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MatPhaseBench evaluates vision-language models on understanding materials phase diagrams, using 200 diagram-text pairs from 3681 papers. It targets complex scientific image understanding, with tasks requiring deep comprehension and open-ended responses, covering 189 material systems and 70 elements.","whyItMatters":"This benchmark addresses the gap in evaluating VLMs on logically complex scientific diagrams that require mechanistic reasoning. It measures capabilities beyond surface perception, helping assess practical value for AI-assisted materials science analysis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6f1bce36a4116ef185ebf2f826a847eb0d532740d3c957c3e61b53082e4ccd79"},"motivation":"Materials phase diagrams are a core knowledge representation in materials science, encoding temperature,composition, phase stability, and phase transformation pathways, with their full understanding requiring thermodynamic mechanism analysis and scientific reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02934","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_ideal-bench_dab673c0","familyId":"bmf_b9aa0b64dd23","name":"IDEAL-Bench","oneLine":"IDEAL-Bench evaluates Vision-Language Models on holistic 3D layout inference from single images of indoor scenes, scoring predictions across five numerical dimensions and a perceptual render-and-compare protocol. It uses a procedurally generated dataset of 1,000 re-renderable Blender environments.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Geometric reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.03614","pdf":"https://arxiv.org/pdf/2607.03614","project":"https://ideal3d.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.03614"},"evidence":{"snippet":"To this end, we introduce IDEAL-Bench, an evaluation suite that requires VLMs to predict structured 3D layouts on photorealistic indoor scenes across 10 room types, scored along five numerical dimensions and a perceptual render-and-compare protocol.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.03614"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"IDEAL-Bench evaluates Vision-Language Models on holistic 3D layout inference from single images of indoor scenes, scoring predictions across five numerical dimensions and a perceptual render-and-compare protocol. It uses a procedurally generated dataset of 1,000 re-renderable Blender environments.","whyItMatters":"Current VLM spatial evaluation relies on question answering, which misses structural understanding. IDEAL-Bench provides a reproducible, quantitative assessment of geometric and structural competencies, revealing model weaknesses in measuring scenes rather than describing them, and offering a diagnostic for genuine spatial intelligence.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3c2efb906e1db2c52ce9292af6a1fcfee7c4a6e02a7b8289ced2d1b156bb41f0"},"motivation":"Spatial question answering is the dominant paradigm for evaluating spatial intelligence in Vision-Language Models (VLMs), but it leaves a complementary axis of spatial competence under-evaluated: holistic 3D layout inference, which predicts every visible object's pose and extent from a single image in a structured form.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.03614","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"IDEAL-Bench Team","organizationType":"academic-lab","sourceUrl":"https://ideal3d.github.io/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_hoosierhelp_ef4e5cac","familyId":"bmf_7c4d86f1a55e","name":"HoosierHelp","oneLine":"HoosierHelp is an interactive benchmark for evaluating LLM agents in social service navigation. Agents interact with simulated users, issue structured resource-search calls, and select final resources from 3,971 Indiana public social service resources. The evaluation focuses on constraint grounding and handling non-ideal user interactions.","area":"Robotics & Embodied AI","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.HC"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.09946","pdf":"https://arxiv.org/pdf/2608.09946","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.09946"},"evidence":{"snippet":"We introduce HoosierHelp, an interactive benchmark grounded in 3,971 Indiana public social service resources.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.09946"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HoosierHelp is an interactive benchmark for evaluating LLM agents in social service navigation. Agents interact with simulated users, issue structured resource-search calls, and select final resources from 3,971 Indiana public social service resources. The evaluation focuses on constraint grounding and handling non-ideal user interactions.","whyItMatters":"Existing benchmarks do not capture the interaction complexity and constraint-grounding demands of social service navigation. HoosierHelp addresses this gap by simulating realistic user behaviors, providing a basis for assessing agent reliability in a high-stakes domain where incorrect resource recommendations can have serious consequences.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7dddef945121a3ef83d8243de18132073fdbbfc3feb16b02440cf1d0f465c4b6"},"motivation":"Social service navigation requires connecting help-seeking individuals to resources that satisfy their needs and specific constraints.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.09946","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"general"},{"id":"bm_guideme_feb34852","familyId":"bmf_f4f8f3df85f0","name":"GuideMe","oneLine":"GuideMe is a benchmark for evaluating multimodal large language models on streaming video task guidance. It includes 2,458 videos (223.7 hours) with 47,775 interaction samples covering next-step instructions, completion feedback, error detection, and corrective guidance. Assessment uses temporal-semantic matching, behavioral classification, and LLM-as-a-Judge.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.02991","pdf":"https://arxiv.org/pdf/2607.02991","project":"https://fawnliu.github.io/project/guideme","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.02991"},"evidence":{"snippet":"In this paper, we construct GuideMe, the first multi-domain benchmark for streaming video that supports training and evaluation of MLLMs for closed-loop interactive task guidance.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02991"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"GuideMe is a benchmark for evaluating multimodal large language models on streaming video task guidance. It includes 2,458 videos (223.7 hours) with 47,775 interaction samples covering next-step instructions, completion feedback, error detection, and corrective guidance. Assessment uses temporal-semantic matching, behavioral classification, and LLM-as-a-Judge.","whyItMatters":"Existing multimodal models lack closed-loop interactive coaching ability; GuideMe provides a standardized evaluation for real-time procedural guidance, highlighting the gap in error detection and corrective feedback, which is crucial for practical assistants.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c6bf1e418be906dd2d54481190c0162801ab27f343cd9276d951faa6cda8e553"},"motivation":"While multimodal Large Language Models (MLLMs) excel at offline video understanding, an interesting question of how far they are from serving as a real-time procedural coach remains unknown.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02991","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GuideMe Project","organizationType":"academic-lab","sourceUrl":"https://fawnliu.github.io/project/guideme","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_gameenginebench_f2b34812","familyId":"bmf_d20da2cff03c","name":"GameEngineBench","oneLine":"GameEngineBench evaluates coding agents on scoped C++ implementation tasks within Unreal Engine 5 projects, built from nine real-world game repositories. The 110 tasks span gameplay, multiplayer, AI, animation, UI, and other areas, requiring native C++ changes that compile and pass behavioral tests.","area":"Code & Software","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Software & Cloud","Manufacturing"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.03525","pdf":"https://arxiv.org/pdf/2607.03525","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.03525"},"evidence":{"snippet":"We present GameEngineBench, a benchmark for evaluating coding agents on scoped C++ implementation tasks inside Unreal Engine 5 projects, built from nine real-world game repositories.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.03525"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GameEngineBench evaluates coding agents on scoped C++ implementation tasks within Unreal Engine 5 projects, built from nine real-world game repositories. The 110 tasks span gameplay, multiplayer, AI, animation, UI, and other areas, requiring native C++ changes that compile and pass behavioral tests.","whyItMatters":"Game-engine development presents unique challenges for coding agents, including stateful, real-time, and interactive systems. This benchmark fills a gap by focusing on deeply integrated C++ tasks, revealing limitations of current agents and providing a practical evaluation for real-world software engineering.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"89ae96020cbf0adb37fa48279cc81e3637acd081e7713d9cfcfcf231c06ad610"},"motivation":"Game engines provide real-time simulation, rendering, physics, interaction, networking, and asset pipelines, making them valuable not only for games but also for 3D applications in healthcare, robotics, architecture, manufacturing, and related domains.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.03525","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GameEngineBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.03525","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_expomotion_345f5881","familyId":"bmf_3b0426612bd6","name":"ExpoMotion","oneLine":"ExpoMotion is a large-scale benchmark for multi-exposure fusion with dynamic scenes, containing 1,738 sequences and 10,909 images across diverse environments. It provides high-fidelity ground truth for reference-based evaluation and a separate set for no-reference evaluation. The benchmark includes training and testing splits with controlled and real-world motions.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.03110","pdf":"https://arxiv.org/pdf/2607.03110","project":null,"code":"https://github.com/Leo-LiuYao/ExpoMotion","data":null,"hfPaper":"https://huggingface.co/papers/2607.03110"},"evidence":{"snippet":"In response, we introduce ExpoMotion, a large-scale benchmark designed to evaluate deghosting capabilities.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.03110"},"ranking":{"90d":{"score":46,"rank":221,"coverage":0.55,"confidence":"Low"}},"description":"ExpoMotion is a large-scale benchmark for multi-exposure fusion with dynamic scenes, containing 1,738 sequences and 10,909 images across diverse environments. It provides high-fidelity ground truth for reference-based evaluation and a separate set for no-reference evaluation. The benchmark includes training and testing splits with controlled and real-world motions.","whyItMatters":"Existing multi-exposure fusion benchmarks often neglect dynamic scenes and lack reliable ground truth, hindering evaluation of deghosting capabilities. ExpoMotion addresses this gap by providing a large-scale dataset with high-quality ground truth, enabling reproducible comparison of methods that handle motion-induced artifacts. This supports practical deployment in real-world scenarios where motion is common.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"13d77869c5a256919370d81a4ccbc50f4705a49fc28cd32745042ea863e6c26e"},"motivation":"Multi-Exposure Fusion (MEF) effectively extends dynamic range, but practical deployment is hindered by motion-induced ghosting and the scarcity of high-quality dynamic benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026","evidence":"Accepted by ECCV 2026","evidenceUrl":"https://arxiv.org/abs/2607.03110","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026","reviewStatus":"accepted","decisionRaw":"Accepted by ECCV 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.03110","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by ECCV 2026","level":"author-claim"}]}],"publishers":[{"name":"ExpoMotion Project","organizationType":"academic-lab","sourceUrl":"https://github.com/Leo-LiuYao/ExpoMotion","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_crackedpdfs_1a52f043","familyId":"bmf_e110c4f10728","name":"CrackedPDFs","oneLine":"Evaluates hidden prompt injection detection in PDFs through classification and paired ranking tasks, using 29,322 generated PDFs from 4,983 base documents. Includes frozen splits, features, and metrics for reproducibility.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19396","pdf":"https://arxiv.org/pdf/2607.19396","project":"https://doi.org/10.5281/zenodo.21735803","code":"https://github.com/volkthienpreecha/crackedpdfs/releases/tag/v1.0.0-paper","data":"https://huggingface.co/datasets/volkthienpreecha/crackedpdfs","hfPaper":"https://huggingface.co/papers/2607.19396"},"evidence":{"snippet":"We introduce CrackedPDFs, a controlled benchmark for hidden prompt injection in PDFs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":241,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2607.19396"},"ranking":{"90d":{"score":27,"rank":385,"coverage":1.0,"confidence":"High","datasetDownloadRank":22,"datasetRankPopulation":51}},"description":"Evaluates hidden prompt injection detection in PDFs through classification and paired ranking tasks, using 29,322 generated PDFs from 4,983 base documents. Includes frozen splits, features, and metrics for reproducibility.","whyItMatters":"Addresses the gap in evaluating defenses against prompt injections embedded in PDF structure, where flattening can hide malicious instructions. Provides a controlled paired benchmark with confounding controls to assess whether detectors generalize beyond superficial cues.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b5aa65e925fc117358b2a0be920f582376b262ebdf182646d53138ebac700c78"},"motivation":"Document-based LLM systems often flatten a PDF before guardrails inspect it.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19396","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_bugsourcebench_7e234238","familyId":"bmf_7f56fcdc5c5d","name":"BugSourceBench","oneLine":"The work introduces BugSourceBench, a code repair benchmark with bugs from human-written, LM-generated, and human-edited LM-generated code. The benchmark evaluates the fix rate of language models on these bug sources. No scoring contract, dataset, or public access path is provided.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.03523","pdf":"https://arxiv.org/pdf/2607.03523","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.03523"},"evidence":{"snippet":"To test whether this curriculum generalizes, we introduce BugSourceBench, a repair benchmark spanning realistic bug sources: bugs in human-written code, LM-generated code, and human-edited LM-generated code.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.03523"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"The work introduces BugSourceBench, a code repair benchmark with bugs from human-written, LM-generated, and human-edited LM-generated code. The benchmark evaluates the fix rate of language models on these bug sources. No scoring contract, dataset, or public access path is provided.","whyItMatters":"Code repair benchmarks often focus on synthetic bugs, which may not reflect real-world failures. BugSourceBench aims to cover diverse bug origins, potentially offering a more realistic evaluation for repair models. However, the lack of accessible artifacts and evaluation protocol limits its current utility.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b9c19a3411cf118e7b6ffe7f130362be00567913ec033ad6be4d7c635e51121f"},"motivation":"Code repair is an important capability for language models (LMs): given a buggy program and unit tests, an LM must produce a fixed program that passes the tests.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.03523","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_wuicc-bench_284cff3d","familyId":"bmf_edbc54d76868","name":"WUICC-bench","oneLine":"WUICC-bench evaluates image change captioning for web UI visual regression testing, using natural language descriptions of UI changes. Scoring includes caption quality metrics and assesses suppression of non-meaningful noise.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.01728","pdf":"https://arxiv.org/pdf/2607.01728","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.01728"},"evidence":{"snippet":"To address the gap, we propose a new task, Web UI Image Change Captioning (WUICC), which sits at the intersection of VRT and image difference captioning (IDC), and release WUICC-bench, its first dataset and benchmark for the task.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01728"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"WUICC-bench evaluates image change captioning for web UI visual regression testing, using natural language descriptions of UI changes. Scoring includes caption quality metrics and assesses suppression of non-meaningful noise.","whyItMatters":"Pixel-level VRT is semantically blind and produces false positives. This benchmark enables development of change captioning systems that describe UI changes in words, improving regression testing efficiency.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"73003b7c5003a2d8108102f09dfc900e16d0f36f07ed622e92a1e5532d2e96ec"},"motivation":"Visual regression testing (VRT) is a standard quality assurance step in modern software release pipelines.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01728","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_underspecbench_dd8281f2","familyId":"bmf_81aeb66da19a","name":"UnderSpecBench","oneLine":"UnderSpecBench evaluates coding agents on DevOps tasks under varying instruction underspecification, measuring action-boundary violations such as wrong-target or over-scope actions.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.02294","pdf":"https://arxiv.org/pdf/2607.02294","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.02294"},"evidence":{"snippet":"We present UnderSpecBench, a benchmark for measuring action-boundary violations in coding agents (i.e., Claude Code, Codex, and OpenCode) on DevOps tasks.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02294"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"UnderSpecBench evaluates coding agents on DevOps tasks under varying instruction underspecification, measuring action-boundary violations such as wrong-target or over-scope actions.","whyItMatters":"Existing agent benchmarks focus on task completion, potentially overstating safe autonomy. UnderSpecBench highlights the gap in measuring safe behavior under underspecified instructions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7a3e3adb24badfc1e0b505199681d83c214c2cf7d2e163d3a14c224eb601b117"},"motivation":"LLM coding agents are increasingly deployed to act autonomously on real production infrastructure.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02294","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_testevo-bench_296298e8","familyId":"bmf_b73f50c8cb68","name":"TestEvo-Bench","oneLine":"TestEvo-Bench evaluates test and code co-evolution tasks from real commit histories. It includes two tracks: test generation (write new tests for changed behavior) and test update (adapt failing tests). Tasks are packaged with environment configurations for execution-grounded metrics like pass rate, coverage, and mutation score. The live component uses timestamps to enable post-training-cutoff evaluation.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.02469","pdf":"https://arxiv.org/pdf/2607.02469","project":"https://www.testevo-bench.com","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.02469"},"evidence":{"snippet":"We introduce TestEvo-Bench, a benchmark of test and code co-evolution tasks mined from software repositories, with two tracks: in test generation, the agent shall write new tests to capture the new software behavior; in test update, the agent shall adapt failing existing tests to the changed software behavior.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02469"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TestEvo-Bench evaluates test and code co-evolution tasks from real commit histories. It includes two tracks: test generation (write new tests for changed behavior) and test update (adapt failing tests). Tasks are packaged with environment configurations for execution-grounded metrics like pass rate, coverage, and mutation score. The live component uses timestamps to enable post-training-cutoff evaluation.","whyItMatters":"Existing benchmarks often separate tests from code changes, relying on static metadata without verifying executability. TestEvo-Bench provides a dynamic, execution-grounded evaluation that reflects real-world test maintenance, enabling assessment of test automation agents in tracking code changes. Its live design reduces data leakage risk, making it valuable for comparing agents in practical settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f56c6e3473f4e16d55190c712fc8c8ec2a11b35ad6ca6e4170c335464af40a5d"},"motivation":"Software tests and code evolve together: a code change should be followed by new or updated tests that record the new software behavior.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02469","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_tca-bench_07c28360","familyId":"bmf_82f0e5522516","name":"TCA-Bench","oneLine":"TCA-Bench is a diagnostic benchmark for evaluating audiovisual binding and temporal relational reasoning in video captioning models, using a decoupled evaluation protocol.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Safety","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.01667","pdf":"https://arxiv.org/pdf/2607.01667","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.01667"},"evidence":{"snippet":"Furthermore, we present TCA-Bench, a diagnostic benchmark utilizing a Decoupled Evaluation Protocol to isolate and quantify model proficiency in audiovisual binding and temporal relational reasoning.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01667"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TCA-Bench is a diagnostic benchmark for evaluating audiovisual binding and temporal relational reasoning in video captioning models, using a decoupled evaluation protocol.","whyItMatters":"It addresses the need for fine-grained evaluation of temporal and cross-modal alignment in audiovisual captioning, offering a protocol to isolate specific model capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0aabbac870ed441d764414f0936488bb2739d0a7f1cec44d73dbfb6e9923b736"},"motivation":"While Multimodal Large Language Models (MLLMs) have advanced video understanding, achieving precise temporal and cross-modal alignment in audiovisual video captioning remains a formidable challenge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01667","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_sovereignnegotiation-bench_d54da144","familyId":"bmf_51f52e6461b2","name":"SovereignNegotiation-Bench","oneLine":"SovereignNegotiation-Bench evaluates personal agents in delegated bargaining scenarios, measuring agreement success alongside user utility, privacy, consent, evidence grounding, concession discipline, escalation, and auditability.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.02814","pdf":"https://arxiv.org/pdf/2607.02814","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.02814"},"evidence":{"snippet":"We introduce SovereignNegotiation-Bench, a trace-level multi-turn benchmark for delegated personal-agent negotiation under private utilities, disclosure constraints, evidence requirements, and institutional asymmetry.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02814"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SovereignNegotiation-Bench evaluates personal agents in delegated bargaining scenarios, measuring agreement success alongside user utility, privacy, consent, evidence grounding, concession discipline, escalation, and auditability.","whyItMatters":"Addresses the gap that existing negotiation benchmarks focus on agreement or surplus, potentially overlooking critical user protections in delegated bargaining. Provides a framework for evaluating both strategic and sovereign aspects of personal agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fe8aefa3ff3e5922523bdf85f62552dfb9c97e438c8650dee517b131314b9553"},"motivation":"Personal agents will increasingly negotiate on behalf of users: splitting costs with other personal agents, appealing platform decisions, escalating support disputes, requesting refunds, changing subscriptions, and negotiating deadlines or reimbursements.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02814","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_pre-flight_898fa920","familyId":"bmf_6f851dc66002","name":"Pre-Flight","oneLine":"Pre-Flight evaluates large language models on aviation operational knowledge via 300 multiple-choice questions drawn from international standards and airport ground operations material, covering ground operations, ICAO and FAA regulations, general aviation knowledge, and operational scenarios. Scoring is by accuracy under a standard multiple-choice protocol using the Inspect framework.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.01829","pdf":"https://arxiv.org/pdf/2607.01829","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.01829"},"evidence":{"snippet":"We present Pre-Flight, an open source benchmark of 300 multiple choice questions drawn from international standards and airport ground operations material, covering international airport ground operations, ICAO and US FAA regulations, aviation general knowledge and complex operational scenarios.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01829"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Pre-Flight evaluates large language models on aviation operational knowledge via 300 multiple-choice questions drawn from international standards and airport ground operations material, covering ground operations, ICAO and FAA regulations, general aviation knowledge, and operational scenarios. Scoring is by accuracy under a standard multiple-choice protocol using the Inspect framework.","whyItMatters":"General-purpose benchmarks do not assess aviation-specific operational safety knowledge, a high-stakes domain where incorrect reasoning can have serious consequences. This benchmark provides a domain-specific evaluation to gauge model reliability for non-safety-critical aviation operations.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"332b08eb0dc49f79dece7a81c9299b4606050beeee64e140a8bdfc4a9c611587"},"motivation":"Large language models (LLMs) are increasingly proposed for aviation business operations, from documentation and training generation to customer facing assistants.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01829","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_patchfusebench_d8b0e871","familyId":"bmf_060068620f54","name":"PatchFuseBench","oneLine":"PatchFuseBench is a fixed-pool benchmark for evaluating repair candidate fusion, built from existing SWE-bench Verified, SWE-bench Multilingual, and Defects4J candidate patches. The benchmark pools candidate patches for 500 bugs on SWE-bench Verified, 300 on Multilingual, and 371 on Defects4J, and evaluates methods that fuse or select a final patch.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.01597","pdf":"https://arxiv.org/pdf/2607.01597","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.01597"},"evidence":{"snippet":"To evaluate this setting, we build PatchFuseBench, a fixed-pool benchmark covering SWE-bench Verified, SWE-bench Multilingual, and Defects4J candidate patches.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01597"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PatchFuseBench is a fixed-pool benchmark for evaluating repair candidate fusion, built from existing SWE-bench Verified, SWE-bench Multilingual, and Defects4J candidate patches. The benchmark pools candidate patches for 500 bugs on SWE-bench Verified, 300 on Multilingual, and 371 on Defects4J, and evaluates methods that fuse or select a final patch.","whyItMatters":"It addresses the pass@k-to-pass@1 gap in code repair, where candidate pools may contain correct patches but selection remains challenging. The benchmark provides a controlled setting to compare post-generation patch selection and fusion methods.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a28f0649ea596e6fecd5fc741ddafdd391dd31066feb15042512db4c67884169"},"motivation":"Modern LLM coding agents are commonly evaluated using pass@k, but developers typically apply a single final patch in real-world settings.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01597","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_opensafeintent_d8b14d02","familyId":"bmf_01b35a219e13","name":"OpenSafeIntent","oneLine":"OpenSafeIntent is a benchmark of controlled prompt-sets that vary user intent while holding the underlying task fixed. Each datapoint contains benign, dual-use, and malicious variants of the same task, and models are evaluated on whether they calibrate assistance across intent shifts rather than only appearing safe on average.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.02047","pdf":"https://arxiv.org/pdf/2607.02047","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.02047"},"evidence":{"snippet":"We introduce OpenSafeIntent, a benchmark of controlled prompt-sets that vary intent while holding the underlying task fixed.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02047"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"OpenSafeIntent is a benchmark of controlled prompt-sets that vary user intent while holding the underlying task fixed. Each datapoint contains benign, dual-use, and malicious variants of the same task, and models are evaluated on whether they calibrate assistance across intent shifts rather than only appearing safe on average.","whyItMatters":"It evaluates safety as intent-calibrated behavior over controlled task variants, addressing limitations of evaluating safety on isolated prompts and providing a way to assess safe completion across subtle intent changes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"59e9506c4747617474f5c97a65f0446bad82e0dc9bb86c1a355d836adf0f3052"},"motivation":"Safe completion requires models to provide useful assistance without enabling harm, but this behavior is difficult to evaluate with isolated prompts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02047","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_mmbench-live_b26d79ee","familyId":"bmf_23818de44e7f","name":"MMBench-Live","oneLine":"MMBench-Live is a continuously evolving multimodal benchmark built by a multi-agent pipeline from MMBench. It contains 5.9K newly generated evaluation instances with executable reasoning, and evaluates vision-language models across question-answer generation and reasoning tasks. Updates cost about USD 30 and take 1-2 hours, with distribution-consistent strategy to maintain comparability.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents","Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.01813","pdf":"https://arxiv.org/pdf/2607.01813","project":null,"code":"https://github.com/PRIS-CV/MMBench-Live","data":null,"hfPaper":"https://huggingface.co/papers/2607.01813"},"evidence":{"snippet":"We present MMBench-Live, a continuously evolving multimodal benchmark built by a multi-agent-driven automated pipeline.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01813"},"ranking":{"90d":{"score":39,"rank":290,"coverage":0.55,"confidence":"Low"}},"description":"MMBench-Live is a continuously evolving multimodal benchmark built by a multi-agent pipeline from MMBench. It contains 5.9K newly generated evaluation instances with executable reasoning, and evaluates vision-language models across question-answer generation and reasoning tasks. Updates cost about USD 30 and take 1-2 hours, with distribution-consistent strategy to maintain comparability.","whyItMatters":"Static benchmarks suffer from contamination and staleness; this benchmark provides a scalable, low-cost paradigm for sustainable evaluation that preserves model rankings and reduces memorization signals.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e87f2bd6c4e4ba1d586557018806e145d3d6044c1d41880947541bb3481736c4"},"motivation":"Evaluation benchmarks are essential for assessing vision-language models (VLMs), but most multimodal benchmarks are static, making them vulnerable to temporal staleness, data contamination, and costly maintenance.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01813","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_medstreambench_73e2ea6c","familyId":"bmf_b4144a83ff7d","name":"MedStreamBench","oneLine":"MedStreamBench is a time-aware benchmark for medical video understanding, integrating 22 medical datasets and 5,419 QA instances across four temporal settings: retrospective, present, future, and proactive. Models are restricted to temporally bounded evidence windows and evaluated on answer correctness, responsiveness, and post-evidence stability.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.01751","pdf":"https://arxiv.org/pdf/2607.01751","project":null,"code":null,"data":"https://huggingface.co/datasets/Venn2024/MedStreamBench","hfPaper":"https://huggingface.co/papers/2607.01751"},"evidence":{"snippet":"We present MedStreamBench, a benchmark for time-aware medical video understanding.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":303,"hfDatasetLikes":4},"source":{"type":"arxiv","id":"2607.01751"},"ranking":{"90d":{"score":55,"rank":167,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":19,"datasetRankPopulation":51}},"description":"MedStreamBench is a time-aware benchmark for medical video understanding, integrating 22 medical datasets and 5,419 QA instances across four temporal settings: retrospective, present, future, and proactive. Models are restricted to temporally bounded evidence windows and evaluated on answer correctness, responsiveness, and post-evidence stability.","whyItMatters":"It addresses the gap between offline recognition and temporally grounded decision-making in clinical settings, where models must decide when to answer or alert. The benchmark provides a protocol for evaluating time-aware reasoning in medical video.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2d8a770354dccdbddfe653a361c7d0b86c6469f65b5308cb13be3d45b315e70d"},"motivation":"Existing medical video benchmarks primarily evaluate whether a model produces the correct answer, but rarely assess whether it answers at the right time.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01751","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_longegorefer_d5a8d99b","familyId":"bmf_b9bf12a8afb9","name":"LongEgoRefer","oneLine":"LongEgoRefer is a benchmark for Video Referring Expression Comprehension in long-form egocentric videos, built from Ego4D. It contains 1,498 referring expressions over videos averaging 45 minutes, requiring temporal grounding of when an occurrence happens and spatial grounding of where the object appears. The benchmark uses evaluation metrics for temporal and spatial grounding.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.02096","pdf":"https://arxiv.org/pdf/2607.02096","project":null,"code":"https://github.com/shunya-kato/LongEgoRefer","data":null,"hfPaper":"https://huggingface.co/papers/2607.02096"},"evidence":{"snippet":"To address this limitation, we introduce LongEgoRefer, a novel and challenging benchmark constructed from long-form videos in the Ego4D dataset.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02096"},"ranking":{"90d":{"score":29,"rank":370,"coverage":0.55,"confidence":"Low"}},"description":"LongEgoRefer is a benchmark for Video Referring Expression Comprehension in long-form egocentric videos, built from Ego4D. It contains 1,498 referring expressions over videos averaging 45 minutes, requiring temporal grounding of when an occurrence happens and spatial grounding of where the object appears. The benchmark uses evaluation metrics for temporal and spatial grounding.","whyItMatters":"Existing egocentric Video REC benchmarks focus on short clips, not reflecting real-world long-form recordings. This benchmark defines a demanding spatio-temporal grounding problem that tests models on sparse object occurrences and complex interactions over extended sequences.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"32c32a2217c1b77e91a22a4a2fe81eee443752aa63cc9fea7294a3cef7677b50"},"motivation":"Egocentric videos capture rich and diverse human-object interactions and have emerged as a fundamental resource for understanding human activities related to objects.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02096","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_k9-bench_cbdfe117","familyId":"bmf_b14e23a954dd","name":"K9-Bench","oneLine":"K9-Bench is a benchmark focused on real-world domestic dog videos, with approximately 5,000 question-answer pairs across 907 videos spanning 5 task categories. The tasks test long-form, canine-centric multimodal reasoning, including action and interaction understanding. A VLM/LLM-powered pipeline was used for data generation.","area":"Multimodal","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.02680","pdf":"https://arxiv.org/pdf/2607.02680","project":"https://ogmenrobotics.github.io/K9Bench","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.02680"},"evidence":{"snippet":"We introduce K9-Bench, a novel benchmark focused on real-world domestic dog videos, specifically targeting canine action and interaction understanding via approximately 5000 question-answer pairs across 907 videos spanning 5 distinct task categories that test long-form, canine-centric multimodal reasoning in MLLMs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02680"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"K9-Bench is a benchmark focused on real-world domestic dog videos, with approximately 5,000 question-answer pairs across 907 videos spanning 5 task categories. The tasks test long-form, canine-centric multimodal reasoning, including action and interaction understanding. A VLM/LLM-powered pipeline was used for data generation.","whyItMatters":"It addresses the underexplored application of multimodal models to animal-centric scenarios, providing a benchmark to evaluate models on recognizing distress signals and enabling responsive robotic companions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c4078098e6a609a824fa7ca61f8db7dfa08f76da36f9547bbbda889098c28934"},"motivation":"MLLMs have shown strong zero-shot capabilities across diverse inputs such as across images, video, audio, and text.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02680","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_javavulbench_93d353dc","familyId":"bmf_c5f6848b3970","name":"JavaVulBench","oneLine":"JavaVulBench is a benchmark dataset and evaluation harness for Java vulnerability detection. The dataset includes about 30,600 Java methods spanning 1,740 CVEs and 700+ projects, with method and line labels, publication dates, and five split strategies. The harness provides a unified schema across multiple backends and includes a contamination audit.","area":"Vision & 3D","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.02825","pdf":"https://arxiv.org/pdf/2607.02825","project":"https://www.youtube.com/watch?v=nMTX\\_hqkuoM","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.02825"},"evidence":{"snippet":"We release \\textsc{JavaVulBench}, a benchmark dataset and evaluation harness for Java vulnerability detection.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02825"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"JavaVulBench is a benchmark dataset and evaluation harness for Java vulnerability detection. The dataset includes about 30,600 Java methods spanning 1,740 CVEs and 700+ projects, with method and line labels, publication dates, and five split strategies. The harness provides a unified schema across multiple backends and includes a contamination audit.","whyItMatters":"It addresses the need for realistic and leakage-aware evaluation of vulnerability detection models, offering multiple split strategies and contamination audits to separate genuinely unseen CVEs from memorized ones.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cb72eb271cd2c494f1b7ccfd68ed2000fda3b7e3a003465535675ada1e918fb7"},"motivation":"We release \\textsc{JavaVulBench}, a benchmark dataset and evaluation harness for Java vulnerability detection.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02825","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_evopolicygym_6d1d11a9","familyId":"bmf_887be45686cc","name":"EvoPolicyGym","oneLine":"EvoPolicyGym evaluates autonomous policy evolution in interactive RL environments. A harness-model agent iteratively edits an executable policy under a fixed interaction budget. The benchmark records trajectories of programs, submissions, feedback, and selection, and scores agents on held-out episodes.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.02440","pdf":"https://arxiv.org/pdf/2607.02440","project":null,"code":"https://github.com/Linzwcs/EvoPolicyGym","data":null,"hfPaper":"https://huggingface.co/papers/2607.02440"},"evidence":{"snippet":"We instantiate this setting in EvoPolicyGym, a benchmark built from compact interactive RL environments that evaluates how agents iteratively improve explored policies.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":51,"hfDailySubmittedAt":null,"githubStars":209,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02440"},"ranking":{"90d":{"score":83,"rank":12,"coverage":0.7,"confidence":"Medium"}},"description":"EvoPolicyGym evaluates autonomous policy evolution in interactive RL environments. A harness-model agent iteratively edits an executable policy under a fixed interaction budget. The benchmark records trajectories of programs, submissions, feedback, and selection, and scores agents on held-out episodes.","whyItMatters":"Existing evaluations often collapse iterative improvement into a final score or confound it with software-engineering progress. EvoPolicyGym isolates the capability to improve policies from bounded feedback, providing trajectory-level diagnostics that distinguish how agents allocate budget and refine policies. This supports comparison of agents on a controlled, reusable protocol.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a657c35593f2b602c149c383fb8440215e4e327a769b92ed67e2939bb4795725"},"motivation":"Autonomous agents are increasingly expected to improve executable policies through feedback, yet existing evaluations often collapse this process into a final score or confound it with open-ended software-engineering progress.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02440","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"EvoPolicyGym Team","organizationType":"academic-lab","sourceUrl":"https://github.com/Linzwcs/EvoPolicyGym","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_eduart_6e0cd321","familyId":"bmf_b8da88c300f6","name":"EduArt","oneLine":"EduArt evaluates art-historical knowledge and visual reasoning in multimodal LLMs using 871 human-authored questions in Italian and English, covering multiple formats and languages. Scoring is based on accuracy and psychometric properties.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Factuality"],"topics":["Multimodal","Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.02007","pdf":"https://arxiv.org/pdf/2607.02007","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.02007"},"evidence":{"snippet":"This paper introduces EduArt, an educational-level benchmark for art-historical knowledge and visual reasoning in multimodal LLMs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02007"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EduArt evaluates art-historical knowledge and visual reasoning in multimodal LLMs using 871 human-authored questions in Italian and English, covering multiple formats and languages. Scoring is based on accuracy and psychometric properties.","whyItMatters":"General benchmarks don't reveal discipline-specific capabilities. EduArt provides a fine-grained benchmark to assess art historical knowledge, showing that format significantly affects performance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"dbb51aece7ec6c9f44b723106dfcb5e8c97989d5a966442965c740fbd4fb9ffc"},"motivation":"Large language models now score near ceiling on general benchmarks, but these aggregate measures reveal little about how models behave within single disciplines.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02007","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_anygroundbench_08e0374e","familyId":"bmf_8d8cdd5ff428","name":"AnyGroundBench","oneLine":"AnyGroundBench evaluates video grounding in vision-language models across five specialized domains (animal, industry, sports, surgery, public security) with spatio-temporal annotations. It provides training and test splits per domain for zero-shot and in-context learning evaluation.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.02269","pdf":"https://arxiv.org/pdf/2607.02269","project":null,"code":"https://github.com/rinost081/AnyGroundBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.02269"},"evidence":{"snippet":"To bridge this gap, we introduce AnyGroundBench, a domain-adaptation benchmark designed to shift the STVG evaluation paradigm from static zero-shot testing to rigorous domain adaptation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":9,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02269"},"ranking":{"90d":{"score":52,"rank":181,"coverage":0.7,"confidence":"Medium"}},"description":"AnyGroundBench evaluates video grounding in vision-language models across five specialized domains (animal, industry, sports, surgery, public security) with spatio-temporal annotations. It provides training and test splits per domain for zero-shot and in-context learning evaluation.","whyItMatters":"Addresses the gap between existing benchmark evaluations on general daily-life videos and real-world specialized applications. Provides a structured domain-adaptation protocol to assess model adaptability in specialized fields, enabling comparative evaluation of VLMs in practical scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f4722e2e2d9b2f384f477de3286a945be6cdf4e3c0a41f2234a454164b93a0b0"},"motivation":"Vision-Language Models (VLMs) have demonstrated immense promise in Spatio-Temporal Video Grounding (STVG).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02269","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Keio University","organizationType":"academic-lab","sourceUrl":"https://github.com/rinost081/AnyGroundBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_agenticdatabench_3427dc68","familyId":"bmf_d78a1404db17","name":"AgenticDataBench","oneLine":"Evaluates LLM-based data agents on realistic data science workflows across 15 domains, with fine-grained ground-truth labels and skill-level scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.DB"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.01647","pdf":"https://arxiv.org/pdf/2607.01647","project":null,"code":"https://github.com/AgenticDataBench/AgenticDataBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.01647"},"evidence":{"snippet":"To address this gap, we propose AgenticDataBench, a comprehensive benchmark featuring realistic tasks spanning diverse domains with fine-grained ground-truth labels.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":37,"hfDailySubmittedAt":null,"githubStars":38,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01647"},"ranking":{"90d":{"score":74,"rank":51,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates LLM-based data agents on realistic data science workflows across 15 domains, with fine-grained ground-truth labels and skill-level scoring.","whyItMatters":"Addresses the lack of comprehensive benchmarks for automating data science workflows, providing fine-grained skill-level insights to guide agent development and selection.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d6074631e8e0d1c9c5df13ee62df0befc8c9731a23deea6cb4565001a4107055"},"motivation":"Data science aims to derive actionable insights from heterogeneous raw data, unlocking the value of the massive amounts of data generated in modern society.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01647","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_yomi-bench_e60b9cad","familyId":"bmf_3edc7fa51add","name":"YOMI-Bench","oneLine":"YOMI-Bench evaluates kanji reading and phonological understanding in LLMs for Japanese through four tasks. It is used in a study assessing multilingual and Japanese-specific models.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.00664","pdf":"https://arxiv.org/pdf/2607.00664","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.00664"},"evidence":{"snippet":"We propose YOMI-Bench, a benchmark for evaluating kanji reading and phonological understanding of large language models (LLMs) for Japanese.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.00664"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"YOMI-Bench evaluates kanji reading and phonological understanding in LLMs for Japanese through four tasks. It is used in a study assessing multilingual and Japanese-specific models.","whyItMatters":"There is no standalone public comparison path or shared artifact; the benchmark primarily supports the paper's finding that LLMs struggle with kanji reading.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"045ed5471782aa578c07d613417991851b229fd20a0c412e9aeae046a55fb502"},"motivation":"We propose YOMI-Bench, a benchmark for evaluating kanji reading and phonological understanding of large language models (LLMs) for Japanese.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.00664","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_rusfinchain_4bba617f","familyId":"bmf_d05efa853bd6","name":"RusFinChain","oneLine":"RusFinChain is a Russian-language benchmark for verifiable chain-of-thought reasoning in finance, comprising 5,280 parameterized examples generated from executable Python templates across 17 domains and 172 topics. Each example includes a gold-standard reasoning chain with intermediate numeric values for automatic verification, and the dataset, code, and evaluation framework are publicly released.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.01388","pdf":"https://arxiv.org/pdf/2607.01388","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.01388"},"evidence":{"snippet":"We present RusFinChain, the first Russian-language symbolic benchmark for verifiable CoT reasoning in finance.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01388"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"RusFinChain is a Russian-language benchmark for verifiable chain-of-thought reasoning in finance, comprising 5,280 parameterized examples generated from executable Python templates across 17 domains and 172 topics. Each example includes a gold-standard reasoning chain with intermediate numeric values for automatic verification, and the dataset, code, and evaluation framework are publicly released.","whyItMatters":"Most financial reasoning benchmarks lack step-level supervision and are English-only; RusFinChain addresses this gap by providing a contamination-free, verifiable CoT benchmark for Russian, enabling automatic assessment of intermediate reasoning steps. It introduces improved metrics that better correlate with final-answer correctness, supporting more diagnostic evaluation of financial LLMs for the Russian-speaking community.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"409d6d6fd0e6d809175c46b08e44cae85ec1fd5294d22ea4caf8a390b53dfdea"},"motivation":"Multi-step symbolic reasoning is essential for robust financial analysis, yet most benchmarks neglect intermediate reasoning steps.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01388","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_phreeqc-mcq-200_2587cac1","familyId":"bmf_b9702619ebb1","name":"PHREEQC-MCQ-200","oneLine":"PHREEQC-MCQ-200 evaluates tool-augmented agents on 200 multiple-choice questions derived from 21 validated PHREEQC scenarios. Agents must construct simulator inputs, execute PHREEQC, inspect structured outputs, and commit to final answers. Scoring is based on exact match of selected answer.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.00436","pdf":"https://arxiv.org/pdf/2607.00436","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.00436"},"evidence":{"snippet":"We introduce PHREEQC-MCQ-200, a benchmark for evaluating tool-augmented agents on deterministic aqueous-geochemistry simulations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.00436"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PHREEQC-MCQ-200 evaluates tool-augmented agents on 200 multiple-choice questions derived from 21 validated PHREEQC scenarios. Agents must construct simulator inputs, execute PHREEQC, inspect structured outputs, and commit to final answers. Scoring is based on exact match of selected answer.","whyItMatters":"This benchmark addresses the lack of standardized evaluation for tool-augmented agents in scientific simulation, measuring not only accuracy but also item-level retention, output-access sensitivity, and trajectory failures. It provides a diagnostic lens on when tool access improves or degrades performance, informing design of reliable scientific agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e49df8fb53660fcbc0cefac963b5825d9e69d2995ad71adc8252578077b8412f"},"motivation":"Large language model agents are increasingly connected to scientific software, yet it remains unclear when tool access makes scientific computation more reliable rather than merely more complex.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.00436","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"The authors","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2607.00436","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_pair-bench_c3cd8d09","familyId":"bmf_7121e1a7ef1c","name":"PAIR-Bench","oneLine":"PAIR-Bench evaluates code improvement by transforming incorrect programs into more correct ones through feedback-guided refinement. It uses progressive hinting with failure-region and hint-depth controls to measure repair of targeted failures, generalization beyond the hint, preservation of correct behavior, and required assistance.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.01360","pdf":"https://arxiv.org/pdf/2607.01360","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.01360"},"evidence":{"snippet":"We introduce PAIR-Bench, a progressive and adaptive benchmark for evaluating code improvement: transforming an incorrect or incomplete program into a more correct one through feedback-guided refinement.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01360"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PAIR-Bench evaluates code improvement by transforming incorrect programs into more correct ones through feedback-guided refinement. It uses progressive hinting with failure-region and hint-depth controls to measure repair of targeted failures, generalization beyond the hint, preservation of correct behavior, and required assistance.","whyItMatters":"Traditional binary pass/fail metrics miss partial progress and refinement trajectories. PAIR-Bench provides finer-grained, progressive metrics to assess how LLMs improve code through feedback, offering practical value for developing and selecting models for code improvement tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7ff056914072d0fa9354c5d0518a824348b6b0efb0313c5b220eb8259131b4de"},"motivation":"Large language models (LLMs) are typically evaluated on code generation and program repair using binary functional correctness: a generated program or patch either passes or fails a test suite.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01360","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_msqa_623c5d9a","familyId":"bmf_96020ebf801d","name":"MSQA","oneLine":"MSQA evaluates 1,064 natively sourced questions across 11 language groups, five cultural dimensions, and three difficulty tiers, testing cultural knowledge in a multilingual context.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.00724","pdf":"https://arxiv.org/pdf/2607.00724","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.00724"},"evidence":{"snippet":"To test this assumption directly, we introduce MSQA, a benchmark of 1,064 natively sourced questions across 11 language groups, five cultural dimensions, and three difficulty tiers.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.00724"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MSQA evaluates 1,064 natively sourced questions across 11 language groups, five cultural dimensions, and three difficulty tiers, testing cultural knowledge in a multilingual context.","whyItMatters":"It addresses the gap in measuring cultural alignment separately from language ability, revealing that models often fail to achieve cultural competence despite multilingual fluency.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f48b33543b788537eecaaea8ef4a4a5e9a9b8975c58f636f3da28457f2ee0d44"},"motivation":"Multilingual fluency often invites a stronger assumption: a model that can speak a user's language must also understand the culture encoded by that language.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.00724","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_molsafeeval_ff6a51a3","familyId":"bmf_d128333456b3","name":"MolSafeEval","oneLine":"MolSafeEval evaluates safety risks in AI-generated molecules across four task types, using structured knowledge graphs and LLM-based reasoning to detect unsafe features.","area":"Safety & Trustworthiness","applicationDomains":["Science & Research","Health & Life Sciences"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals","Pharma & Biotech"],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.00464","pdf":"https://arxiv.org/pdf/2607.00464","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.00464"},"evidence":{"snippet":"To address this gap, we introduce MolSafeEval, a benchmark dedicated to evaluating and analyzing the safety risks of molecular generation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.00464"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MolSafeEval evaluates safety risks in AI-generated molecules across four task types, using structured knowledge graphs and LLM-based reasoning to detect unsafe features.","whyItMatters":"Fills a gap in molecular generation benchmarks by focusing on safety risks, providing standardized datasets and protocols for safer molecular design.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"110ba94d96f9b54c7265804078c5d2318e72423dcd38b2914ca6ecb9b3ef59ea"},"motivation":"Current molecular generation benchmarks emphasize task complexity, molecule novelty, and property alignment; they largely overlook a critical concern: the potential safety risks of AI-generated molecules.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"Findings of ACL 2026","evidence":"Accepted by Findings of ACL 2026","evidenceUrl":"https://arxiv.org/abs/2607.00464","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"Findings of ACL 2026","reviewStatus":"accepted","decisionRaw":"Accepted by Findings of ACL 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.00464","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by Findings of ACL 2026","level":"author-claim"}]}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"cross-domain"},{"id":"bm_mohallbench_4062fe02","familyId":"bmf_4bf5148e0196","name":"MoHallBench","oneLine":"MoHallBench evaluates motion hallucination in video LLMs with 11,306 video clips and 40,493 QA pairs, covering three hallucination sources and multiple choice settings.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.01117","pdf":"https://arxiv.org/pdf/2607.01117","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.01117"},"evidence":{"snippet":"We present MoHallBench, a benchmark for diagnosing motion hallucination in VideoLLMs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01117"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MoHallBench evaluates motion hallucination in video LLMs with 11,306 video clips and 40,493 QA pairs, covering three hallucination sources and multiple choice settings.","whyItMatters":"Targets a specific video understanding failure mode, offering metrics to reduce affirmation bias and revealing gaps in action recognition vs. hallucination resistance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6e809959fb7db64d9322a2f884fdf69d38628c2724cc2878fc814cd47cf48147"},"motivation":"Video Large Language Models (VideoLLMs) have shown strong progress in video understanding, yet they still suffer from hallucinations that are inconsistent with visual evidence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01117","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mindedit-bench_02b6e953","familyId":"bmf_ebce6cd9eb55","name":"MindEdit-Bench","oneLine":"MindEdit-Bench evaluates counterfactual spatial reasoning in VLMs with 1,003 multiple-choice questions from private indoor scenes, covering six task types.","area":"Vision & 3D","applicationDomains":["Consumer & Productivity"],"primaryDomain":"Consumer & Productivity","industrySectors":["Consumer Technology"],"capabilities":["Reasoning","Geometric reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.00491","pdf":"https://arxiv.org/pdf/2607.00491","project":null,"code":null,"data":"https://huggingface.co/datasets/ZODAOfficial/MindEdit-Bench","hfPaper":"https://huggingface.co/papers/2607.00491"},"evidence":{"snippet":"We introduce MindEdit-Bench, a benchmark of six spatial reasoning tasks built from three-photo smartphone triplets of newly captured indoor scenes via an automatic in-the-wild 3D scene-graph extraction pipeline.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":160,"hfDatasetLikes":1},"source":{"type":"arxiv","id":"2607.00491"},"ranking":{"90d":{"score":46,"rank":226,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":26,"datasetRankPopulation":51}},"description":"MindEdit-Bench evaluates counterfactual spatial reasoning in VLMs with 1,003 multiple-choice questions from private indoor scenes, covering six task types.","whyItMatters":"Tests whether VLMs can reason about hypothetical object manipulations, a capability not covered by existing benchmarks, with human-verified answers.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d54d6d1a7d263f5a4b25aa43486eee2c5b6fccc75f219a1ad4181db3e06a7e67"},"motivation":"Benchmarks for vision-language models (VLMs) mostly test observational spatial reasoning: models describe relations already visible in the input.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.00491","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ZODAOfficial","organizationType":"community","sourceUrl":"https://huggingface.co/datasets/ZODAOfficial/MindEdit-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_memsyco-bench_f1a6a168","familyId":"bmf_3d36e1cca960","name":"MemSyco-Bench","oneLine":"MemSyco-Bench evaluates memory-induced sycophancy in agent systems across five tasks, measuring how memory influences decision-making and personalization.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.01071","pdf":"https://arxiv.org/pdf/2607.01071","project":null,"code":"https://github.com/XMUDeepLIT/MemSyco-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2607.01071"},"evidence":{"snippet":"To bridge this gap, we propose MemSyco-Bench, a comprehensive benchmark for evaluating memory-induced sycophancy in agent systems.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":31,"hfDailySubmittedAt":null,"githubStars":17,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01071"},"ranking":{"90d":{"score":68,"rank":87,"coverage":0.7,"confidence":"Medium"}},"description":"MemSyco-Bench evaluates memory-induced sycophancy in agent systems across five tasks, measuring how memory influences decision-making and personalization.","whyItMatters":"Addresses a gap in memory benchmarks by focusing on downstream reasoning effects, offering a leaderboard and standardized evaluation code.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"73a3d5d893eb00c0bf467fe671cc3ad6247bd0f6ac298ddfc89ed82a5f1bd692"},"motivation":"Memory has emerged as a cornerstone of modern LLM-based agents, supporting their evolution from single-turn assistants to long-term collaborators.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01071","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"XMUDeepLIT","organizationType":"academic-lab","sourceUrl":"https://github.com/XMUDeepLIT/MemSyco-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_longvqubench_aca1b92b","familyId":"bmf_324133a6cb63","name":"LongVQUBench","oneLine":"LongVQUBench evaluates long-term video quality understanding with 1200+ videos and 1500 questions across three levels of perceptual reasoning complexity.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.01086","pdf":"https://arxiv.org/pdf/2607.01086","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.01086"},"evidence":{"snippet":"To address these limitations, we present LongVQUBench, a comprehensive benchmark for long-term video quality understanding.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01086"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"LongVQUBench evaluates long-term video quality understanding with 1200+ videos and 1500 questions across three levels of perceptual reasoning complexity.","whyItMatters":"Fills a gap in video quality benchmarks by focusing on temporal continuity and cumulative degradation, with hierarchical evaluation levels for systematic assessment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ae62241396a6b7b64ede5b38df18a685936f72b9367d6c6ba0c8ce7aaeec0893"},"motivation":"The evaluation of long-term video quality understanding remains an open challenge for large vision-language models (LVLMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"European Conference on Computer Vision 2026","evidence":"Accepted at European Conference on Computer Vision 2026","evidenceUrl":"https://arxiv.org/abs/2607.01086","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"European Conference on Computer Vision 2026","reviewStatus":"accepted","decisionRaw":"Accepted at European Conference on Computer Vision 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.01086","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at European Conference on Computer Vision 2026","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_llvm-bench_e576b85e","familyId":"bmf_efd956bb75ce","name":"LLVM-Bench","oneLine":"LLVM-Bench evaluates LLMs on resolving LLVM compiler issues with 423 real-world tasks, using an automated evaluation platform.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.00700","pdf":"https://arxiv.org/pdf/2607.00700","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.00700"},"evidence":{"snippet":"To address this gap, we introduce LLVM-Bench, the first large-scale benchmark for LLVM issue resolution, containing 423 real-world, validated tasks collected from the LLVM project.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.00700"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LLVM-Bench evaluates LLMs on resolving LLVM compiler issues with 423 real-world tasks, using an automated evaluation platform.","whyItMatters":"Provides a large-scale benchmark for system-level compiler issue resolution, addressing a gap in LLM evaluation for complex software engineering tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b1f821a5a5dbd4c5977eb5fb437718f38e0fe2fe36d66dff4608d75c857b5ebb"},"motivation":"LLVM is a widely used compiler infrastructure whose scale and complexity make issue resolution labor-intensive and challenging.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.00700","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_isosci_576d9a68","familyId":"bmf_c5ac3d50efe8","name":"IsoSci","oneLine":"IsoSci evaluates reasoning vs. knowledge retrieval in LLMs using isomorphic cross-domain problem pairs, enabling controlled attribution of reasoning gains.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Information retrieval","Factuality"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.01431","pdf":"https://arxiv.org/pdf/2607.01431","project":null,"code":null,"data":"https://huggingface.co/datasets/isosci/isosci","hfPaper":"https://huggingface.co/papers/2607.01431"},"evidence":{"snippet":"We introduce ISOSCI, a benchmark of isomorphic cross-domain science problem pairs that separates reasoning ability from domain knowledge retrieval in LLM evaluation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":15,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2607.01431"},"ranking":{"90d":{"score":36,"rank":304,"coverage":0.3,"confidence":"Low","datasetDownloadRank":50,"datasetRankPopulation":51}},"description":"IsoSci evaluates reasoning vs. knowledge retrieval in LLMs using isomorphic cross-domain problem pairs, enabling controlled attribution of reasoning gains.","whyItMatters":"Provides a method to separate reasoning ability from knowledge, challenging assumptions about chain-of-thought and guiding model evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"76a48fd05695f3d246651a97e75b6d54882ec2995b71eb870668244065323ede"},"motivation":"We introduce ISOSCI, a benchmark of isomorphic cross-domain science problem pairs that separates reasoning ability from domain knowledge retrieval in LLM evaluation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01431","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_egogapbench_95dd2183","familyId":"bmf_db4933aac951","name":"EgoGapBench","oneLine":"EgoGapBench evaluates egocentric action selection in multi-agent scenes, isolating the ability to choose actions from the agent's perspective when other agents are present. The benchmark includes training and test splits with human performance as reference.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2607.00547","pdf":"https://arxiv.org/pdf/2607.00547","project":null,"code":"https://github.com/jhCOR/EgoGapBench","data":null,"hfPaper":"https://huggingface.co/papers/2607.00547"},"evidence":{"snippet":"To isolate egocentric perspective understanding, we introduce EgoGapBench, a diagnostic benchmark for measuring action selection in multi-agent egocentric scenes.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.00547"},"ranking":{"90d":{"score":29,"rank":371,"coverage":0.55,"confidence":"Low"}},"description":"EgoGapBench evaluates egocentric action selection in multi-agent scenes, isolating the ability to choose actions from the agent's perspective when other agents are present. The benchmark includes training and test splits with human performance as reference.","whyItMatters":"Existing egocentric benchmarks conflate first-person view processing with perspective-taking, making it hard to isolate perspective understanding. EgoGapBench fills this gap by providing a controlled evaluation for a capability that is crucial for embodied AI and human-robot interaction, showing that state-of-the-art models fail at this task.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"406666ebd38ef0445abfd398a8a28da2642cd2c92a5c7e54796d1d4ff9fdd3c3"},"motivation":"Existing egocentric benchmarks have primarily constructed the egocentric setting from first-person-view data, which makes it difficult to evaluate egocentric perspective itself in isolation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.00547","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"jhCOR","organizationType":"academic-lab","sourceUrl":"https://github.com/jhCOR/EgoGapBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_cruisebench_5f43e5a1","familyId":"bmf_3ac2a92019a7","name":"CruiseBench","oneLine":"CruiseBench evaluates remaining useful life prediction for aircraft engines using a fixed cruise-stage protocol derived from N-CMAPSS. It provides a reproducible sub-benchmark with datasets and baseline results for LSTM, GRU, TCN, and TSMixer models under specific settings.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19380","pdf":"https://arxiv.org/pdf/2607.19380","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19380"},"evidence":{"snippet":"To mitigate this issue, this paper proposes CruiseBench, a cruise-stage RUL benchmark derived from N-CMAPSS.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19380"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CruiseBench evaluates remaining useful life prediction for aircraft engines using a fixed cruise-stage protocol derived from N-CMAPSS. It provides a reproducible sub-benchmark with datasets and baseline results for LSTM, GRU, TCN, and TSMixer models under specific settings.","whyItMatters":"RUL prediction is critical for maintenance planning, but existing benchmarks like N-CMAPSS lack evaluation control due to full-flight records. CruiseBench offers a controlled, reproducible setting for fair comparison of RUL models, reducing variability from stage and preprocessing choices.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"91d7d5726156b546179187a0567da6519e6cdabefa27cbcd57a71e5a8688fce1"},"motivation":"Remaining useful life (RUL) prediction estimates how long an engine can continue safe operation and is central to maintenance planning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19380","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_clarifycodebench_aa8d2946","familyId":"bmf_76530bd5505f","name":"ClarifyCodeBench","oneLine":"ClarifyCodeBench evaluates LLMs on clarifying ambiguous requirements for code generation through interactive dialogues. It includes manual annotations of ambiguity types, clarification questions, and ground-truth answers, with metrics for interaction quality.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation"],"topics":["Code"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.00711","pdf":"https://arxiv.org/pdf/2607.00711","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.00711"},"evidence":{"snippet":"To bridge this gap, we introduce ClarifyCodeBench, a novel interactive benchmark for evaluating LLMs' capability in resolving requirement ambiguity.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.00711"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ClarifyCodeBench evaluates LLMs on clarifying ambiguous requirements for code generation through interactive dialogues. It includes manual annotations of ambiguity types, clarification questions, and ground-truth answers, with metrics for interaction quality.","whyItMatters":"Code generation in practice involves underspecified requirements, but existing benchmarks assume perfect prompts. ClarifyCodeBench addresses this by measuring a critical yet underexplored capability, revealing that strong code generation does not imply effective requirement clarification.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"58ded43d2b3f7996b63a9725c4d6d88976d54d37f9a558b36f34b6a3ab737f7c"},"motivation":"Large Language Models have emerged as programming assistants.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.00711","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_chainswe_b0efa40a","familyId":"bmf_9b5273a86b4c","name":"ChainSWE","oneLine":"ChainSWE evaluates coding agents on sequential, dependent bug fixes within a shared codebase. It includes 304 issues across 54 Python projects, forming chronological chains, and measures performance drop as chain length increases.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.02606","pdf":"https://arxiv.org/pdf/2607.02606","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.02606"},"evidence":{"snippet":"To bridge this gap, we introduce ChainSWE, the first benchmark for evaluating agents on sequential, dependent bug fixes within a shared codebase.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.02606"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ChainSWE evaluates coding agents on sequential, dependent bug fixes within a shared codebase. It includes 304 issues across 54 Python projects, forming chronological chains, and measures performance drop as chain length increases.","whyItMatters":"Real-world software maintenance involves streams of related defects, but existing benchmarks evaluate one bug at a time. ChainSWE fills this gap by benchmarking agents on continuous workflows, revealing significant performance degradation on longer chains.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"57a51864fdbc645474213d93abaa4481ad903fabf6bc64fef5231455c5c3f4d1"},"motivation":"Language model (LM) agents are increasingly deployed to maintain codebases over extended periods, fixing streams of related defects while carrying context from one fix to the next.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.02606","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_av-syncbench_20190a72","familyId":"bmf_e3adb616e734","name":"AV-SyncBench","oneLine":"AV-SyncBench evaluates audio-visual synchronization by separating temporal and semantic consistency. It contains 3,269 videos and 38,390 samples across 10 scenarios and 5 tasks, with data filtered and verified for on-screen sound sources.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.00726","pdf":"https://arxiv.org/pdf/2607.00726","project":"https://fgt7t6g.github.io/AV-SyncBench","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.00726"},"evidence":{"snippet":"We propose AV-SyncBench, the first benchmark to fully separate temporal and semantic evaluation for audio-visual synchronization.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.00726"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AV-SyncBench evaluates audio-visual synchronization by separating temporal and semantic consistency. It contains 3,269 videos and 38,390 samples across 10 scenarios and 5 tasks, with data filtered and verified for on-screen sound sources.","whyItMatters":"Existing AV feature extraction evaluations are coupled, preventing independent assessment of temporal and semantic alignment. AV-SyncBench provides a decoupled benchmark to quantify feature quality for alignment and downstream tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"34dda38356605d3475f21bd7a31babfcb03e38ef2bf133e7362bcc77ae5f0f2a"},"motivation":"Audio-visual feature extraction is a fundamental component of multimodal understanding and generation tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"Interspeech 2026","evidence":"Accepted by Interspeech 2026","evidenceUrl":"https://arxiv.org/abs/2607.00726","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"Interspeech 2026","reviewStatus":"accepted","decisionRaw":"Accepted by Interspeech 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.00726","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by Interspeech 2026","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_agc-bench_28cf2135","familyId":"bmf_8f34ce070e53","name":"AGC-Bench","oneLine":"AGC-Bench evaluates artificial general creativity across 78 datasets covering brainstorming, problem solving, STEM, narrative, figurative language, and humor. It uses an agentic harness and a public leaderboard with an open-weight judge model.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-07-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.01152","pdf":"https://arxiv.org/pdf/2607.01152","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.01152"},"evidence":{"snippet":"We introduce AGC-Bench, an artificial general creativity benchmark built from a systematic review of the AI creativity literature (3,101 papers screened, 497 benchmarks identified), paired with an agentic harness that converts idiosyncratic codebases into HELM-standardized benchmarks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.01152"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AGC-Bench evaluates artificial general creativity across 78 datasets covering brainstorming, problem solving, STEM, narrative, figurative language, and humor. It uses an agentic harness and a public leaderboard with an open-weight judge model.","whyItMatters":"Creativity is a key aspect of intelligence often overlooked in AI evaluation. AGC-Bench provides a comprehensive, standardized measurement of AI creativity, revealing distinct strengths and weaknesses across models and a single creativity factor analogous to general intelligence.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dd67d7d201c57369233f4f7573d29aacdfb2b30bc95a1bd759c228e22bc41d54"},"motivation":"Creativity research has debated whether creativity is domain-specific (e.g., visual, writing, science), and if it is psychometrically separable from general intelligence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.01152","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_worldroambench_ba7d4093","familyId":"bmf_0d99b2d6873b","name":"WorldRoamBench","oneLine":"WorldRoamBench evaluates interactive world models on long-horizon stability across action, vision, physics, and memory dimensions. It includes 600+ test cases in various scenes and views with continuous interaction.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.31672","pdf":"https://arxiv.org/pdf/2606.31672","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.31672"},"evidence":{"snippet":"We introduce WorldRoamBench, an open-world benchmark for long-horizon stability across four dimensions, each with tailored innovations: (i) Action: per-frame action metric bypassing cross-model semantic scale disparity and exposing failures hidden by trajectory; (ii) Vision: segment-based drift metric capturing non-monotonic mid-sequence collapse missed by start-vs-end comparisons; (iii) Physics: controllability-gated evaluation over mechanics, optics, and 3D consistency, scoring plausibility un","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31672"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"WorldRoamBench evaluates interactive world models on long-horizon stability across action, vision, physics, and memory dimensions. It includes 600+ test cases in various scenes and views with continuous interaction.","whyItMatters":"Interactive world models need stable, physically grounded, and memory-faithful behavior, but existing benchmarks ignore these aspects. WorldRoamBench provides a comprehensive evaluation revealing that no current model reliably satisfies all dimensions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0151c8c8bd0d6bf6a0a8a303bcdef18953be429dbe8bd12944a57e965bb33af5"},"motivation":"Despite rapid progress in interactive world models (IWMs), existing benchmarks evaluate action following only at trajectory level and ignore memory and interaction physics.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31672","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_vulbench-cpp_84620b34","familyId":"bmf_12ecf4f9c98a","name":"VULBENCH-CPP","oneLine":"VULBENCH-CPP evaluates the safety of AI-generated C++ code using multi-tier verification including functional testing, static analysis, dynamic analysis, and bounded model checking. It includes 8,918 programs from three LLMs and human authors.","area":"Safety & Trustworthiness","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":["Code generation"],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.00107","pdf":"https://arxiv.org/pdf/2607.00107","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.00107"},"evidence":{"snippet":"We introduce VULBENCH-CPP, a benchmark of 8,918 C++ programs from three open-weight LLMs (Gemma 3 27B IT, LLaMA 3.3 70B Instruct, Qwen 2.5 Coder 32B Instruct) and human authors across 851 competitive-programming tasks.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.00107"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"VULBENCH-CPP evaluates the safety of AI-generated C++ code using multi-tier verification including functional testing, static analysis, dynamic analysis, and bounded model checking. It includes 8,918 programs from three LLMs and human authors.","whyItMatters":"Security of AI-generated code is critical, but evaluations often rely on a single method. VULBENCH-CPP provides a comprehensive multi-tier benchmark that reveals AI code is more prone to runtime violations and that no single verification tier is sufficient.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"261920b3c0aa5b0e431cd835e8c27561c77eadfbb2f834c9e0caaa2a72e39042"},"motivation":"As large language models (LLMs) are increasingly deployed for systems programming, their ability to generate secure C++ code, where a single memory-safety failure creates an exploitable vulnerability, remains a critical concern.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.00107","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness","Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_sense-vad_052128b1","familyId":"bmf_a69f1f20096d","name":"SENSE-VAD","oneLine":"SENSE-VAD is a synthetic video anomaly detection dataset for autonomous driving, generated with CARLA and Unreal Engine. It includes socially complex anomalies across five categories with per-frame binary labels, plus real-world videos for sim-to-real transfer.","area":"Vision & 3D","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Automotive"],"capabilities":[],"topics":["cs.CV"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.31875","pdf":"https://arxiv.org/pdf/2606.31875","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.31875"},"evidence":{"snippet":"We introduce SENSE-VAD, the first synthetic video anomaly detection benchmark for autonomous driving explicitly designed around socially complex anomalies.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31875"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SENSE-VAD is a synthetic video anomaly detection dataset for autonomous driving, generated with CARLA and Unreal Engine. It includes socially complex anomalies across five categories with per-frame binary labels, plus real-world videos for sim-to-real transfer.","whyItMatters":"Addresses the evaluation gap for socially complex anomalies in autonomous driving, which are not captured by motion-based detectors. Provides a controlled benchmark to test current video anomaly detection models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"558112397f630815a69a6b85c4bfbbdd057e1e2c89c0dcf69cd808cd9ee19940"},"motivation":"Autonomous vehicles (AVs) must navigate not only motion-based hazards but also socially complex situations whose danger is constituted by inter-agent relationships rather than movement statistics alone.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31875","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_scratchworld_04c08f86","familyId":"bmf_0395c14a6789","name":"ScratchWorld","oneLine":"ScratchWorld is an offline diagnostic benchmark for world models, using Scratch projects as executable worlds. It evaluates next-state prediction, long-horizon tracking, causal attribution, and counterfactual prediction with replay-verified transitions.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.31689","pdf":"https://arxiv.org/pdf/2606.31689","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.31689"},"evidence":{"snippet":"We introduce ScratchWorld, an offline diagnostic benchmark that treats Scratch projects as executable worlds and uses a pinned Scratch VM to produce replay-verified transitions, hidden variables, causal traces, and counterfactual outcomes.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31689"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ScratchWorld is an offline diagnostic benchmark for world models, using Scratch projects as executable worlds. It evaluates next-state prediction, long-horizon tracking, causal attribution, and counterfactual prediction with replay-verified transitions.","whyItMatters":"Provides a new evaluation paradigm for world models that avoids confounds like overlap with persistent state. Enables direct comparison of model capabilities on executable consequences.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"20ec2b91a1225ff4419b7bcc3e7d4df8f4a8ece6b5522b0356cb2d3390f43c93"},"motivation":"World-model evaluations often score a predicted future by overlap with a target state or observation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31689","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_qval_38cf43e7","familyId":"bmf_77df7a8b1bd4","name":"QVal","oneLine":"QVal is a training-free testbed for evaluating dense supervision signals for long-horizon LLM agents. It measures Q-alignment of scores from methods across four environments and seven families, without requiring training runs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.32034","pdf":"https://arxiv.org/pdf/2606.32034","project":null,"code":"https://github.com/bethgelab/qval","data":null,"hfPaper":"https://huggingface.co/papers/2606.32034"},"evidence":{"snippet":"We introduce QVal, a training-free testbed for directly evaluating dense supervision signals.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":12,"hfDailySubmittedAt":null,"githubStars":8,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.32034"},"ranking":{"90d":{"score":60,"rank":135,"coverage":0.7,"confidence":"Medium"}},"description":"QVal is a training-free testbed for evaluating dense supervision signals for long-horizon LLM agents. It measures Q-alignment of scores from methods across four environments and seven families, without requiring training runs.","whyItMatters":"Fills the gap of direct, comparable evaluation of dense supervision methods, which is currently expensive and confounded. Enables early-stage assessment before training.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9cbee184baa666518cbe2891e72f220a38a9e4986850de93aedb917bf7ca30a5"},"motivation":"LLM agents increasingly act over long horizons, where a single trajectory can contain hundreds or thousands of actions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.32034","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"bethgelab","organizationType":"academic-lab","sourceUrl":"https://github.com/bethgelab/qval","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_ppt-eval_9317e9e6","familyId":"bmf_32aec5f2ab84","name":"PPT-Eval","oneLine":"PPT-Eval is a benchmark of 120 PowerPoint tasks across 12 files for computer-use agents. It covers content creation and editing, with rubric-based evaluation that awards partial credit and provides natural language feedback.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.31154","pdf":"https://arxiv.org/pdf/2606.31154","project":"https://microsoft.github.io/ppteval","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.31154"},"evidence":{"snippet":"We introduce PPT-Eval, a benchmark of 120 PowerPoint tasks across 12 files that cover both content creation and presentation editing scenarios, organized by difficulty.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31154"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PPT-Eval is a benchmark of 120 PowerPoint tasks across 12 files for computer-use agents. It covers content creation and editing, with rubric-based evaluation that awards partial credit and provides natural language feedback.","whyItMatters":"Provides a realistic, multimodal testbed for computer-use agents. The rubric-based scoring captures partial progress and correlates with human judgment, enabling nuanced comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5a27f5023da02e575c3b0c42ea5523c37770d4f14da9bd97bdf8f660b8fd44ab"},"motivation":"Creating and editing slides is a rich, multimodal activity that is ubiquitous in professional and educational settings, making it an ideal testbed for real-world computer-use agents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31154","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Microsoft","organizationType":"company-research-lab","sourceUrl":"https://microsoft.github.io/ppteval","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_mecobench_fd729c4e","familyId":"bmf_a28f8f826055","name":"MECoBench","oneLine":"MECoBench is a multimodal embodied cooperation benchmark with an evaluation platform. It spans real-world tasks, two cooperation structures, and three collaboration modes, with code and dataset publicly available.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Agents","Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.31966","pdf":"https://arxiv.org/pdf/2606.31966","project":"https://q-i-n-g.github.io/MECoBench-Website/","code":"https://github.com/q-i-n-g/MECoBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.31966"},"evidence":{"snippet":"To address this gap, we introduce MECoBench, a multimodal embodied cooperation benchmark with an evaluation platform spanning diverse real-world tasks, two cooperation structures, and three collaboration modes.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31966"},"ranking":{"90d":{"score":49,"rank":206,"coverage":0.55,"confidence":"Low"}},"description":"MECoBench is a multimodal embodied cooperation benchmark with an evaluation platform. It spans real-world tasks, two cooperation structures, and three collaboration modes, with code and dataset publicly available.","whyItMatters":"Systematically evaluates collaboration among multimodal embodied agents, a relatively unexplored area. Provides a testbed for understanding collaboration mechanisms and limits.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d96a96bde9c953799192146b51b052286595c4cecd1518fec4286fed54d9273e"},"motivation":"Recent multimodal large language models (MLLMs) have strong potential as embodied agents, but their ability to collaborate in visually grounded environments remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31966","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"q-i-n-g","organizationType":"academic-lab","sourceUrl":"https://github.com/q-i-n-g/MECoBench","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_jl1-cc-qa_e1b89c2a","familyId":"bmf_0a1d39bb805e","name":"JL1-CC&QA","oneLine":"JL1-CC&QA is a multi-task benchmark extending JL1-CD with change captioning and question answering. It includes 17,021 captions and 20,060 QA pairs over 5,000 bi-temporal image pairs from Jilin-1 satellite.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.31745","pdf":"https://arxiv.org/pdf/2606.31745","project":null,"code":"https://github.com/circleLZY/JL1-CD","data":null,"hfPaper":"https://huggingface.co/papers/2606.31745"},"evidence":{"snippet":"To bridge this semantic gap, we introduce JL1-CC&QA, a multi-task benchmark that extends the JL1-CD dataset with two complementary annotation layers: change captioning (CC) and change question answering (QA).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":128,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31745"},"ranking":{"90d":{"score":75,"rank":49,"coverage":0.55,"confidence":"Low"}},"description":"JL1-CC&QA is a multi-task benchmark extending JL1-CD with change captioning and question answering. It includes 17,021 captions and 20,060 QA pairs over 5,000 bi-temporal image pairs from Jilin-1 satellite.","whyItMatters":"Bridges the semantic gap in remote sensing change detection by providing captions and QA alongside binary masks. Enables multi-task understanding of surface changes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3bc6f9c58a01af83d374dcfeeddec50588500ff8840730c1af07bf7040025633"},"motivation":"Remote sensing change detection (CD) traditionally focuses on pixel-level binary segmentation, which identifies where changes occur but neither what nor why.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31745","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"circleLZY","organizationType":"academic-lab","sourceUrl":"https://github.com/circleLZY/JL1-CD","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_jeto-bench_f10e93b0","familyId":"bmf_714867b9d42b","name":"JETO-Bench","oneLine":"JETO-Bench is a benchmark of 660 execution time improvement patches (ETIPs) in Java, with 91 manually verified. It is built using JETO-Mine, a configurable tool that mines ETIPs from GitHub repositories and statistically validates improvements.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.31767","pdf":"https://arxiv.org/pdf/2606.31767","project":null,"code":"https://github.com/khesoem/JETO-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.31767"},"evidence":{"snippet":"Using JETO-Mine, we build JETO-Bench, a benchmark of 660 identified and 91 manually verified executable ETIPs from 174 Java repositories, mined from nearly 1.8 million commits across 11 years.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31767"},"ranking":{"90d":{"score":39,"rank":291,"coverage":0.55,"confidence":"Low"}},"description":"JETO-Bench is a benchmark of 660 execution time improvement patches (ETIPs) in Java, with 91 manually verified. It is built using JETO-Mine, a configurable tool that mines ETIPs from GitHub repositories and statistically validates improvements.","whyItMatters":"Provides a reproducible benchmark for performance bug fixing in Java, an area lacking benchmarks. Enables evaluation of patch generation and test generation tools.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"94cc332a0e494acb03efaa68cb0a40813a28fbca709b4e6782b667f0addeab8c"},"motivation":"Automated fixing of performance issues is gaining attention, but existing benchmarks of execution time improvement patches (ETIPs) target Python, C++, or .NET and are fixed datasets that cannot be extended under user-defined configurations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31767","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"khesoem","organizationType":"academic-lab","sourceUrl":"https://github.com/khesoem/JETO-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_interflopbench_d649e042","familyId":"bmf_601ef5f2c623","name":"InterFLOPBench","oneLine":"InterFLOPBench is a benchmark of 90 C kernels and 1,130 test samples for evaluating LLMs on floating-point error classification across six categories: cancellation, comparison, division by zero, overflow, underflow, and NaN.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.31308","pdf":"https://arxiv.org/pdf/2606.31308","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.31308"},"evidence":{"snippet":"We introduce InterFLOPBench, a benchmark of 90 C kernels with 1 130 test samples designed to evaluate LLMs across six categories of floating-point error: cancellation, comparison, division by zero, overflow, underflow and NaN, compared across 14 LLMs.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31308"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"InterFLOPBench is a benchmark of 90 C kernels and 1,130 test samples for evaluating LLMs on floating-point error classification across six categories: cancellation, comparison, division by zero, overflow, underflow, and NaN.","whyItMatters":"Provides a targeted evaluation for LLM capabilities in static floating-point error detection, a niche but important area. Enables comparison across models and error types.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b18394eee69131223a87b869faa349657c075afa90557d84f88fb0cd3a002a4e"},"motivation":"This paper investigates the capability of Large Language Models (LLMs) to detect and classify floating-point errors statically in software code.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31308","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_healthagentbench_400912dd","familyId":"bmf_38ba71a1d0f7","name":"HealthAgentBench","oneLine":"Evaluates AI agents on 54 realistic healthcare tasks across 7 categories, including medical imaging, EHR analysis, and clinical trial matching. Agents operate in terminal environments with task-specific verifiers and a final task success rate.","area":"Agents & Tool Use","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.31179","pdf":"https://arxiv.org/pdf/2606.31179","project":null,"code":"https://github.com/microsoft/HealthAgentBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.31179"},"evidence":{"snippet":"We introduce HealthAgentBench, a suite of 54 agentic healthcare tasks across 7 categories each with its unique environment.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":null,"githubStars":41,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31179"},"ranking":{"90d":{"score":72,"rank":66,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates AI agents on 54 realistic healthcare tasks across 7 categories, including medical imaging, EHR analysis, and clinical trial matching. Agents operate in terminal environments with task-specific verifiers and a final task success rate.","whyItMatters":"Provides a standardized, realistic evaluation for agentic healthcare AI, revealing performance gaps across task types and informing model selection for clinical applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"65afc00a2c6c32b161975e9b50e9fdf9dcd8514bb9fac5e2b1ae58b65a1b4129"},"motivation":"As AI agents become increasingly capable of complex, long-horizon reasoning, rigorous and holistic evaluation is essential for measuring progress toward real-world healthcare applications.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31179","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Microsoft","organizationType":"company-research-lab","sourceUrl":"https://github.com/microsoft/HealthAgentBench","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_finpersona-bench_6543120b","familyId":"bmf_47d6e24194de","name":"FinPersona-Bench","oneLine":"Evaluates the longitudinal stability of behavioral mandates in LLM-based financial agents using a synthetic market simulation that decouples observable price from hidden fundamental value, scoring mandate adherence across calm, crash, and bubble market regimes.","area":"Agents & Tool Use","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.CL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.31522","pdf":"https://arxiv.org/pdf/2606.31522","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.31522"},"evidence":{"snippet":"To measure MSD objectively, we introduce FinPersona-Bench, a simulation benchmark in which a synthetic market decouples observable price from hidden fundamental value, enabling falsifiable evaluation across three failure modes: trading without signal in calm markets, panic-selling during crashes, and ignoring fundamental value during speculative bubbles.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31522"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates the longitudinal stability of behavioral mandates in LLM-based financial agents using a synthetic market simulation that decouples observable price from hidden fundamental value, scoring mandate adherence across calm, crash, and bubble market regimes.","whyItMatters":"It addresses the gap in evaluating long-horizon behavioral consistency of autonomous financial agents, providing a falsifiable method to measure mandate salience decay and informing deployment decisions for mandate-aware re-grounding strategies.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4dcfae791689a136b0d677536f5d291ecc206f601cbad44aa895d432c954ac8c"},"motivation":"Large Language Models (LLMs) are increasingly deployed as autonomous financial agents initialized with explicit behavioral mandates such as \"preserve capital\" or \"avoid speculative bets\" that are meant to govern every decision throughout deployment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31522","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_egosafetybench_4d21ada3","familyId":"bmf_4c45392c5812","name":"EgoSafetyBench","oneLine":"EgoSafetyBench is a diagnostic egocentric video benchmark of 1,200 robot-view scenarios to evaluate vision-language models as runtime safety guards. It assesses situational awareness across routine, suspicious, obvious, and contextual hazards, and visual-channel robustness against misleading in-scene text.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.00218","pdf":"https://arxiv.org/pdf/2607.00218","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.00218"},"evidence":{"snippet":"We introduce EgoSafetyBench, an egocentric video benchmark of 1,200 robot-view scenarios annotated at half-second granularity, to evaluate VLMs as streaming guards across two tracks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.00218"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"EgoSafetyBench is a diagnostic egocentric video benchmark of 1,200 robot-view scenarios to evaluate vision-language models as runtime safety guards. It assesses situational awareness across routine, suspicious, obvious, and contextual hazards, and visual-channel robustness against misleading in-scene text.","whyItMatters":"This benchmark addresses the practical need for safety guards that distinguish genuine hazards from superficially alarming but benign actions, and highlights the vulnerability to misleading signage, which can inform safer deployment of embodied VLMs in real-world settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"98c489eb2d4216f5b04646e759a36801531ad2874cb0d51bf52f0fc32cd8c5a9"},"motivation":"Vision-language models (VLMs) are now proposed as runtime safety guards for embodied agents in homes and factories.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.00218","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_clawarena-team_8c93773c","familyId":"bmf_c7eb98430ee5","name":"ClawArena-Team","oneLine":"ClawArena-Team evaluates a single text-only LLM's ability to manage a fixed, locally served pool of subagents (LLM, VLM, omni) across 41 multi-turn, multimodal, multi-directory scenarios with 258 evaluation rounds and 72 staged updates. Scoring is execution-based via shell commands, producing a Subagent-Management Score (SMS) that multiplies task correctness by a least-privilege and modality-routing factor, without LLM judges.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.31174","pdf":"https://arxiv.org/pdf/2606.31174","project":"https://www.clawarena.cc/","code":"https://github.com/aiming-lab/ClawArena","data":null,"hfPaper":"https://huggingface.co/papers/2606.31174"},"evidence":{"snippet":"We introduce ClawArena-Team, a benchmark of 41 multi-turn, multimodal, multi-directory scenarios spanning 258 evaluation rounds and 72 staged updates that measures this management ability.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":61,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31174"},"ranking":{"90d":{"score":72,"rank":64,"coverage":0.55,"confidence":"Low"}},"description":"ClawArena-Team evaluates a single text-only LLM's ability to manage a fixed, locally served pool of subagents (LLM, VLM, omni) across 41 multi-turn, multimodal, multi-directory scenarios with 258 evaluation rounds and 72 staged updates. Scoring is execution-based via shell commands, producing a Subagent-Management Score (SMS) that multiplies task correctness by a least-privilege and modality-routing factor, without LLM judges.","whyItMatters":"Existing agent benchmarks measure a policy's own task-solving or emergent behavior of fixed multi-agent systems, but not the leadership capability of a single model orchestrating subagents. ClawArena-Team fills this gap by isolating management skill from raw capability, supporting decisions on model selection for delegation-heavy workflows and revealing cost-quality trade-offs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9666a2ac4bc96807405ca733dd9d6cc3d5253f3cac5e46e4dde649dd478f78fc"},"motivation":"Production large language-model (LLM) agents are increasingly deployed not as lone problem-solvers but as managers: a main model creates specialized subagents, delegates work, and orchestrates their parallel, asynchronous returns through dynamic workflows.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31174","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_cdr-bench_40384718","familyId":"bmf_11d5499469aa","name":"CDR-Bench","oneLine":"CDR-Bench is a benchmark of 3,462 tasks for evaluating large language models on faithful execution of compositional, order-sensitive data refinement recipes across four domains and 29 operators, with deterministic reference outputs for exact evaluation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.31435","pdf":"https://arxiv.org/pdf/2606.31435","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.31435"},"evidence":{"snippet":"To fill this gap, we introduce CDR-Bench, a comprehensive benchmark featuring 3,462 high-quality tasks spanning four real-world data refinement domains and 29 distinct operators.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.31435"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"CDR-Bench is a benchmark of 3,462 tasks for evaluating large language models on faithful execution of compositional, order-sensitive data refinement recipes across four domains and 29 operators, with deterministic reference outputs for exact evaluation.","whyItMatters":"Existing benchmarks leave unclear whether LLMs can directly execute multi-step, order-sensitive data refinement tasks; this benchmark provides a reusable, deterministic evaluation protocol to assess procedural faithfulness in compositional text processing.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b1a757d03ed27beb6ebe96b3552827493948e0a11eda8b9d3c56bfe6770ea86b"},"motivation":"Data refinement involves executing multi-step recipes over evolving text states, where both composition and execution order of processing operators determine the outcome.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.31435","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_swe-together_260e2683","familyId":"bmf_123ecb3dd733","name":"SWE-Together","oneLine":"SWE-Together evaluates coding agents in multi-turn interactive user sessions reconstructed from real user-agent interactions. It comprises 109 repository-level tasks with a reactive LLM-based user simulator, measuring final repository correctness and the number of corrective feedback turns.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Agents"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29957","pdf":"https://arxiv.org/pdf/2606.29957","project":null,"code":"https://github.com/Togetherbench/SWE-Together","data":null,"hfPaper":"https://huggingface.co/papers/2606.29957"},"evidence":{"snippet":"We introduce SWE-Together, a multi-turn benchmark reconstructed from real user-agent coding sessions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":14,"hfDailySubmittedAt":null,"githubStars":56,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29957"},"ranking":{"90d":{"score":76,"rank":40,"coverage":0.7,"confidence":"Medium"}},"description":"SWE-Together evaluates coding agents in multi-turn interactive user sessions reconstructed from real user-agent interactions. It comprises 109 repository-level tasks with a reactive LLM-based user simulator, measuring final repository correctness and the number of corrective feedback turns.","whyItMatters":"Existing coding-agent benchmarks often evaluate static, single-turn tasks, missing the interactive nature of real coding assistance. SWE-Together provides a reproducible protocol for assessing agents as collaborators, capturing both task success and user effort, offering practical value for comparing agents in realistic settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a2b28d10dee2abdbcde94ad261c709e00c47c052a270c697ee14dce0e07a2dc2"},"motivation":"Most coding-agent benchmarks are static: an agent receives a complete task description up front and is judged only by its final code.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.29957","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Togetherbench","organizationType":"community","sourceUrl":"https://github.com/Togetherbench/SWE-Together","role":"benchmark-publisher"}],"capabilityGroups":["Agents","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_swe-interact_046bbdbf","familyId":"bmf_93acd7a96f92","name":"SWE-INTERACT","oneLine":"SWE-Interact evaluates coding agents on multi-turn, interactive software engineering tasks where a simulated user provides vague instructions, reveals requirements progressively, and gives feedback. The benchmark comprises 75 tasks and measures agents' ability to discover user intent, adapt to evolving requirements, and build on prior work.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Code"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.30573","pdf":"https://arxiv.org/pdf/2606.30573","project":null,"code":"https://github.com/scaleapi/SWE-Interact","data":null,"hfPaper":"https://huggingface.co/papers/2606.30573"},"evidence":{"snippet":"We introduce SWE-Interact, a new testbed for evaluating coding agents on multi-turn, interactive, user-driven software engineering tasks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":null,"githubStars":24,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.30573"},"ranking":{"90d":{"score":68,"rank":85,"coverage":0.7,"confidence":"Medium"}},"description":"SWE-Interact evaluates coding agents on multi-turn, interactive software engineering tasks where a simulated user provides vague instructions, reveals requirements progressively, and gives feedback. The benchmark comprises 75 tasks and measures agents' ability to discover user intent, adapt to evolving requirements, and build on prior work.","whyItMatters":"Existing SWE benchmarks focus on single-turn autonomous implementation, but real developer workflows are interactive. SWE-Interact fills the gap by measuring performance on long-horizon, user-driven tasks, showing that strong single-turn performance does not reliably transfer. This provides a more realistic evaluation axis for coding agents and guides development of models that can collaborate effectively with users.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f9fc79903ecb54517711cf6a755aafb2701ab8e2f8bdce223851471f86fae8cb"},"motivation":"We introduce SWE-Interact, a new testbed for evaluating coding agents on multi-turn, interactive, user-driven software engineering tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.30573","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Scale AI","organizationType":"company-research-lab","sourceUrl":"https://github.com/scaleapi/SWE-Interact","role":"benchmark-publisher"}],"capabilityGroups":["Agents","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_spreadsheetbench_a0b75616","familyId":"bmf_15499180d75b","name":"SpreadsheetBench","oneLine":"SpreadsheetBench 2 evaluates spreadsheet agents on end-to-end business workflows across generation, debugging, and visualization tasks. It includes 321 tasks from authentic business data, with multi-sheet workbooks requiring cross-sheet reasoning. The benchmark provides a unified multi-turn agent scaffold and evaluation scripts for reproducibility.","area":"Code & Software","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29955","pdf":"https://arxiv.org/pdf/2606.29955","project":"https://spreadsheetbench.github.io/","code":"https://github.com/RUCKBReasoning/SpreadsheetBench-2","data":null,"hfPaper":"https://huggingface.co/papers/2606.29955"},"evidence":{"snippet":"We introduce \\textsc{SpreadsheetBench 2}, a workflow-level benchmark for spreadsheet agents that covers three task categories: generation, debugging, and visualization.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":25,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29955"},"ranking":{"90d":{"score":61,"rank":125,"coverage":0.7,"confidence":"Medium"}},"description":"SpreadsheetBench 2 evaluates spreadsheet agents on end-to-end business workflows across generation, debugging, and visualization tasks. It includes 321 tasks from authentic business data, with multi-sheet workbooks requiring cross-sheet reasoning. The benchmark provides a unified multi-turn agent scaffold and evaluation scripts for reproducibility.","whyItMatters":"Existing spreadsheet benchmarks focus on isolated operations, failing to capture real-world workflow complexity. SpreadsheetBench 2 addresses this gap by assessing agents on tasks that require multi-step coordination, cross-sheet reasoning, and deliverable-level outcomes. It provides a challenging testbed for improving reliable spreadsheet automation, with current models achieving only 34.89% overall accuracy.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"329d728aee104cb1c8c7d0c29699f8dd02c46b3c2b39565d5c320a36736ba71b"},"motivation":"Spreadsheets are widely used for business analysis, financial modeling, reporting, and decision-making.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.29955","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"RUCKBReasoning","organizationType":"academic-lab","sourceUrl":"https://github.com/RUCKBReasoning/SpreadsheetBench-2","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"specific"},{"id":"bm_shovir_1cca6b61","familyId":"bmf_72ae455766ea","name":"SHOVIR","oneLine":"SHOVIR evaluates vision shortcut learning in radiology report generation by extending MIMIC-CXR and PadChest-GR with per-box CheXpert labels. It defines image-level and disease-level occlusion experiments that compare model predictions on clean images against localized perturbations to isolate direct and contextual shortcut failures at the disease-class level.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CV"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.30201","pdf":"https://arxiv.org/pdf/2606.30201","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.30201"},"evidence":{"snippet":"We introduce SHOVIR, a benchmark for evaluating vision shortcut behavior in RRG.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.30201"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SHOVIR evaluates vision shortcut learning in radiology report generation by extending MIMIC-CXR and PadChest-GR with per-box CheXpert labels. It defines image-level and disease-level occlusion experiments that compare model predictions on clean images against localized perturbations to isolate direct and contextual shortcut failures at the disease-class level.","whyItMatters":"Standard RRG metrics miss whether diagnostic statements are grounded in actual image evidence, allowing models to exploit dataset priors. SHOVIR provides a protocol to assess spatial grounding, revealing that high report quality can coexist with shallow visual reliance, which is critical for clinical deployment decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"222d6f65b9d787c3b317be7a0386db381e68803608001c41ba22a0832d825ce7"},"motivation":"Current evaluation protocols for Vision-Language Models (VLMs) in Radiology Report Generation (RRG) rely on report-level metrics that measure lexical overlap or aggregate clinical correctness.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.30201","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_sciir-bench_0a7d1ae8","familyId":"bmf_e6067af2051a","name":"SciIR-Bench","oneLine":"SciIR-Bench evaluates text-to-image models on scientific image reasoning across three semiotic-aligned tracks: entity structure, scientific process, and scientific law. It uses an atomic checklist to convert scientific accuracy into verifiable fine-grained questions.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.30124","pdf":"https://arxiv.org/pdf/2606.30124","project":null,"code":"https://github.com/MAIR-Lab-HUST/SciIR","data":null,"hfPaper":"https://huggingface.co/papers/2606.30124"},"evidence":{"snippet":"For evaluation, we propose SciIR-Bench, which aligns with these three semiotic levels and employs an Atomic Checklist to convert the outcome-oriented scientific accuracy into process-oriented, verifiable, fine-grained questions.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":null,"githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.30124"},"ranking":{"90d":{"score":56,"rank":158,"coverage":0.7,"confidence":"Medium"}},"description":"SciIR-Bench evaluates text-to-image models on scientific image reasoning across three semiotic-aligned tracks: entity structure, scientific process, and scientific law. It uses an atomic checklist to convert scientific accuracy into verifiable fine-grained questions.","whyItMatters":"Current text-to-image models lack rigorous evaluation for scientific imagery, which requires logical reasoning beyond visual fidelity. SciIR-Bench provides a structured protocol to measure such capabilities, aiding selection and development of models for scientific visualization tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"83d731dbb3f312a2f5e2092f2b709ab7f0034d8f9dd4ec634cb4c2f8520ef193"},"motivation":"While Text-to-Image (T2I) models have shown remarkable success in generating photorealistic visual content, they still struggle with the rigorous semantic alignment and logical reasoning required for scientific imagery.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026","evidence":"Accepted to ECCV 2026","evidenceUrl":"https://arxiv.org/abs/2606.30124","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026","reviewStatus":"accepted","decisionRaw":"Accepted to ECCV 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.30124","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to ECCV 2026","level":"author-claim"}]}],"publishers":[{"name":"MAIR-Lab-HUST","organizationType":"academic-lab","sourceUrl":"https://github.com/MAIR-Lab-HUST/SciIR","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_safepyramid_fe239f10","familyId":"bmf_efe4e93686e2","name":"SafePyramid","oneLine":"SafePyramid evaluates in-context policy guardrailing across 1,000 multi-turn conversations and 3,000 application-specific policies containing 61,699 natural-language rules, organized into three hierarchical capability levels (L0, L1, L2). Scoring is based on violated-rule set prediction with metrics RMR and RDR, and the evaluation harness supports any API or local model.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29887","pdf":"https://arxiv.org/pdf/2606.29887","project":null,"code":"https://github.com/bytedance/safepyramid","data":null,"hfPaper":"https://huggingface.co/papers/2606.29887"},"evidence":{"snippet":"To systematically evaluate this capability, we introduce SafePyramid, a safety benchmark comprising 1,000 multi-turn conversations across 10 domains and 3,000 corresponding application-specific policies, which together contain 61,699 distinct natural-language rules.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":6,"hfDailySubmittedAt":null,"githubStars":8,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29887"},"ranking":{"90d":{"score":58,"rank":140,"coverage":0.7,"confidence":"Medium"}},"description":"SafePyramid evaluates in-context policy guardrailing across 1,000 multi-turn conversations and 3,000 application-specific policies containing 61,699 natural-language rules, organized into three hierarchical capability levels (L0, L1, L2). Scoring is based on violated-rule set prediction with metrics RMR and RDR, and the evaluation harness supports any API or local model.","whyItMatters":"SafePyramid addresses the gap in evaluating guardrails under application-specific policies rather than fixed taxonomies, providing a structured test for rule understanding, dependency resolution, and adaptation to novel frameworks. It enables direct comparison of frontier LLMs and configurable guardrails on a realistic safety task.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"29270bd355d906f7bc22eec279f2e1c76f1c24b56f454878457e62a4f07f1159"},"motivation":"In real-world applications, guardrails are often expected to identify unsafe user-model interactions according to application-specific safety policies, rather than relying on predefined risk taxonomies.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.29887","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ByteDance","organizationType":"company-research-lab","sourceUrl":"https://github.com/bytedance/safepyramid","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_saber-math_7bfa1c5b","familyId":"bmf_7856a310f2fa","name":"SABER-Math","oneLine":"SABER-Math evaluates information retrieval for mathematical queries, with about 283K problems and tasks for reranking based on fine-grained relevance. Scoring uses preference tournament ratings.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.IR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29894","pdf":"https://arxiv.org/pdf/2606.29894","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.29894"},"evidence":{"snippet":"We address this gap by introducing SABER-Math, the first fully automated benchmark for evaluating mathematical IR without expert annotation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29894"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SABER-Math evaluates information retrieval for mathematical queries, with about 283K problems and tasks for reranking based on fine-grained relevance. Scoring uses preference tournament ratings.","whyItMatters":"Existing IR benchmarks fail to capture mathematical relevance, and MTEB doesn't predict math performance. SABER-Math provides a math-specific benchmark to guide retriever selection.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"547f8647ce984c70f7184dd5f84253559a73559fe639920ee0986bfac8515a8f"},"motivation":"As agentic AI systems tackle more complex mathematical tasks, they increasingly rely on information retrieval (IR) to search problem databases, theorem libraries, and educational resources.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"in the 3rd AI for Math Workshop at the 43rd International Conference on Machine Learning (ICML), Seoul, South Korea, 202","evidence":"Accepted in the 3rd AI for Math Workshop at the 43rd International Conference on Machine Learning (ICML), Seoul, South Korea, 2026","evidenceUrl":"https://arxiv.org/abs/2606.29894","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"in the 3rd AI for Math Workshop at the 43rd International Conference on Machine Learning (ICML), Seoul, South Korea, 202","reviewStatus":"accepted","decisionRaw":"Accepted in the 3rd AI for Math Workshop at the 43rd International Conference on Machine Learning (ICML), Seoul, South Korea, 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.29894","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted in the 3rd AI for Math Workshop at the 43rd International Conference on Machine Learning (ICML), Seoul, South Korea, 2026","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval","Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_ruverbench_289e3a74","familyId":"bmf_646ce3fcec4f","name":"RuVerBench","oneLine":"RuVerBench evaluates LLM-as-a-judge reliability for rubric verification in agentic scenarios. It includes 2,458 instances across deep research and agentic coding, each with a model-generated output, a rubric, and a human-annotated label indicating rubric satisfaction.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29920","pdf":"https://arxiv.org/pdf/2606.29920","project":null,"code":"https://github.com/THU-KEG/RuVerBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.29920"},"evidence":{"snippet":"We introduce RuVerBench, the first benchmark for assessing LaaJ reliability in rubric verification for agentic scenarios.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":9,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29920"},"ranking":{"90d":{"score":57,"rank":151,"coverage":0.7,"confidence":"Medium"}},"description":"RuVerBench evaluates LLM-as-a-judge reliability for rubric verification in agentic scenarios. It includes 2,458 instances across deep research and agentic coding, each with a model-generated output, a rubric, and a human-annotated label indicating rubric satisfaction.","whyItMatters":"Rubric-based scoring with LLM judges is common but under-validated, especially for agentic outputs. RuVerBench provides a reusable benchmark to compare judge models and strategies, enabling decisions on model selection and scoring protocol.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"20987ca85eee3fb2da929edc4f10fd99d9ae46bfb1b19dfa6a19e2dd7b656988"},"motivation":"Rubric-based scoring has become a widely used paradigm in model evaluation, typically with LLM-as-a-Judge (LaaJ) for rubric scoring.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.29920","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Tsinghua University","organizationType":"academic-lab","sourceUrl":"https://github.com/THU-KEG/RuVerBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_repair-bench_816a6914","familyId":"bmf_cd82db311a00","name":"REPAIR-Bench","oneLine":"REPAIR-Bench evaluates robot error perception and recovery in human-robot interaction. It includes 214 interaction trials from 41 participants with four induced failure types, synchronized facial action units, head pose, speech transcripts, and post-interaction reports. Three tasks cover failure detection across sessions, failure-type classification, and recovery prediction.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29937","pdf":"https://arxiv.org/pdf/2606.29937","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.29937"},"evidence":{"snippet":"We present REPAIR-Bench, built on 214 interaction trials from 41 participants, the benchmark spans four induced failure types and provides synchronized facial action units, head pose, speech transcripts, and post-interaction affect and recovery reports.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29937"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"REPAIR-Bench evaluates robot error perception and recovery in human-robot interaction. It includes 214 interaction trials from 41 participants with four induced failure types, synchronized facial action units, head pose, speech transcripts, and post-interaction reports. Three tasks cover failure detection across sessions, failure-type classification, and recovery prediction.","whyItMatters":"REPAIR-Bench addresses the lack of unified benchmarks for HRI failures, enabling standardized evaluation of failure detection, classification, and recovery prediction. This supports the development of adaptive and trustworthy robot systems and provides a comparison baseline for future research.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f9dfa802988a6a09d0afd85dfbb22788fe1b83946008ad8ae198a1c97f92b50b"},"motivation":"Understanding how users perceive and respond to robot failures is essential for building robust and trustworthy robot systems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.29937","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_oao-attackbench_8518a31c","familyId":"bmf_ebc7ad501506","name":"OAO-AttackBench","oneLine":"OAO-AttackBench is a set of counterfactual prompts for one-and-only objects, used to evaluate text-to-image alignment in a specific study.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.30262","pdf":"https://arxiv.org/pdf/2606.30262","project":"https://soyoun-won.github.io/one-and-only-ir-guidance/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.30262"},"evidence":{"snippet":"To systematically evaluate the challenging task of aligning generative outputs with unusual prompts for OAO objects, we introduce OAO-AttackBench, a benchmark comprising counterfactual prompts that directly conflict with the core visual identity of OAO objects.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.30262"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OAO-AttackBench is a set of counterfactual prompts for one-and-only objects, used to evaluate text-to-image alignment in a specific study.","whyItMatters":"The benchmark supports evaluating prompt adherence for concepts with strong visual priors, but it is introduced primarily to validate the proposed method and lacks a standalone public comparison path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"61373e2c58bd1fe4bbc74ed2b2e2d3c1f6d3d5864bbc12c1a1e72bfc57210870"},"motivation":"Text-to-image (T2I) diffusion models often fail to faithfully render explicit textual descriptions, instead defaulting to strongly learned visual priors due to a phenomenon referred to as concept association bias.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026","evidence":"Accepted at ECCV 2026","evidenceUrl":"https://arxiv.org/abs/2606.30262","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026","reviewStatus":"accepted","decisionRaw":"Accepted at ECCV 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.30262","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at ECCV 2026","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_nl-pddl-bench_3b0f6c71","familyId":"bmf_57cc06c1d191","name":"NL-PDDL-Bench","oneLine":"NL-PDDL-Bench evaluates natural-language-to-PDDL specification generation. It consists of multi-domain instances from IPC domains with planner-verified executability, difficulty scaled by object count, and a suite for parseability, solvability, and plan-level consistency.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.29700","pdf":"https://arxiv.org/pdf/2606.29700","project":null,"code":"https://github.com/ibasicplan/NL-PDDL-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.29700"},"evidence":{"snippet":"We present NL-PDDL-Bench, a multi-domain benchmark for natural-language-to-PDDL specification construction with planner-verified executability and controlled difficulty scaling by object count.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29700"},"ranking":{"90d":{"score":39,"rank":292,"coverage":0.55,"confidence":"Low"}},"description":"NL-PDDL-Bench evaluates natural-language-to-PDDL specification generation. It consists of multi-domain instances from IPC domains with planner-verified executability, difficulty scaled by object count, and a suite for parseability, solvability, and plan-level consistency.","whyItMatters":"This benchmark addresses the lack of standardized evaluation for LLM-generated planning specifications, with an emphasis on executability and verifiability. It provides a reproducible basis for assessing model reliability in safety-sensitive planning applications, where incorrect formalization can lead to unsafe outcomes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4f45b825d0fb9bfbde2b16b0422ddd99354885540e2916e519ecc955809c66c9"},"motivation":"Planning often requires symbolic specifications that are both executable and verifiable.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.29700","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ibasicplan","organizationType":"academic-lab","sourceUrl":"https://github.com/ibasicplan/NL-PDDL-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_musebench_7c930c72","familyId":"bmf_a54402b9054c","name":"MuseBench","oneLine":"MuseBench evaluates multimodal large language models on intent-level understanding of audiovisual arts, covering cinematic arts, static visual arts, stage performing arts, and game arts with 4,016 questions in single- and multi-select formats.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.30026","pdf":"https://arxiv.org/pdf/2606.30026","project":"https://musebench.github.io","code":"https://github.com/musebench/musebench-code","data":null,"hfPaper":"https://huggingface.co/papers/2606.30026"},"evidence":{"snippet":"To address this gap, we introduce Musebench, a comprehensive benchmark designed to evaluate MLLMs on nuanced artistic understanding.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":6,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.30026"},"ranking":{"90d":{"score":54,"rank":173,"coverage":0.7,"confidence":"Medium"}},"description":"MuseBench evaluates multimodal large language models on intent-level understanding of audiovisual arts, covering cinematic arts, static visual arts, stage performing arts, and game arts with 4,016 questions in single- and multi-select formats.","whyItMatters":"Existing video and multimodal benchmarks mostly test perceptual recognition, leaving artistic intent and creative reasoning unevaluated. MuseBench measures a distinct capability gap, providing a reference for progress in creative-domain expertise.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"02efd189ac6f4f0d72cba16da7eff21a3083aee28aff4b4f3be33048ddf67c96"},"motivation":"Audiovisual arts encompass diverse creative disciplines, including cinema, visual arts, stage performance, and game design, where artistic meaning arises from deliberate combinations of visual, auditory, and narrative elements (e.g., fear amplified through claustrophobic framing, or grief conveyed through silence and lingering close-ups).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.30026","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MuseBench project","organizationType":"academic-lab","sourceUrl":"https://github.com/musebench/musebench-code","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mirrorcode_114f6eb8","familyId":"bmf_5f4db04ac40d","name":"MirrorCode","oneLine":"MirrorCode evaluates AI agents on reimplementing entire software projects from behavior only, matching outputs on end-to-end tests across 25 programs spanning Unix utilities, data serialization, bioinformatics, interpreters, static analysis, cryptography, and compression.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.30182","pdf":"https://arxiv.org/pdf/2606.30182","project":null,"code":"https://github.com/epoch-research/MirrorCode","data":null,"hfPaper":"https://huggingface.co/papers/2606.30182"},"evidence":{"snippet":"To address these challenges, we introduce MirrorCode, a long-horizon coding benchmark based on reimplementing entire software projects.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":66,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.30182"},"ranking":{"90d":{"score":73,"rank":59,"coverage":0.55,"confidence":"Low"}},"description":"MirrorCode evaluates AI agents on reimplementing entire software projects from behavior only, matching outputs on end-to-end tests across 25 programs spanning Unix utilities, data serialization, bioinformatics, interpreters, static analysis, cryptography, and compression.","whyItMatters":"Existing coding benchmarks focus on shorter tasks, while long-horizon reimplementation remains hard to compare. MirrorCode provides a standardized, repeatable measure of autonomous software engineering capability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"67c61e915e20b607b728481872859b99bae0e3b00f5681543f13e73b2ec5d59e"},"motivation":"AI models are rapidly improving at autonomous coding, as shown by benchmark progress and one-off demonstrations such as AI implementing a C compiler.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.30182","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Epoch Research","organizationType":"academic-lab","sourceUrl":"https://github.com/epoch-research/MirrorCode","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_madi-bench_da5b9728","familyId":"bmf_1f7a76f278ff","name":"MaDI-Bench","oneLine":"MaDI-Bench evaluates end-to-end data integration pipelines across schema matching, value normalization, entity matching, and conflict resolution.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.DB"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.30371","pdf":"https://arxiv.org/pdf/2606.30371","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.30371"},"evidence":{"snippet":"This paper fills this gap by introducing the Mannheim Data Integration Benchmark (MaDI-Bench), the first benchmark for the end-to-end integration of relational tables covering all steps of the integration process.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.30371"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MaDI-Bench evaluates end-to-end data integration pipelines across schema matching, value normalization, entity matching, and conflict resolution.","whyItMatters":"Data integration tasks are often evaluated piecemeal; an end-to-end benchmark could support comparison of holistic pipelines.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"366f7d1acdf55fabedacaf4f7c76759de4264be29cc1ecaed331cbeabb2d14af"},"motivation":"Data integration combines heterogeneous data sets into a single, coherent representation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.30371","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_empath_eb9c1b58","familyId":"bmf_85697bf1d578","name":"EMPATH","oneLine":"EMPATH evaluates safety of emotional-support chatbots via auditor-generated multi-turn conversations scored on 19 metrics across crisis handling, therapeutic quality, conversational integrity, emotional safety, and cultural adaptation.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.30256","pdf":"https://arxiv.org/pdf/2606.30256","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.30256"},"evidence":{"snippet":"We present EMPATH, a benchmark for safety evaluation of emotional-support chatbots.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.30256"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EMPATH evaluates safety of emotional-support chatbots via auditor-generated multi-turn conversations scored on 19 metrics across crisis handling, therapeutic quality, conversational integrity, emotional safety, and cultural adaptation.","whyItMatters":"Safety evaluation for emotional-support chatbots needs multilingual, multi-turn metrics; EMPATH appears to address that gap.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8c0e01b95d534c5c4990283bc510140cad34ead7a421e06e73b6d5c6b673f5bd"},"motivation":"Safety benchmarks often buy scalability by fixing the prompt, the language, and the turn structure.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.30256","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_citbench_cd2dd37c","familyId":"bmf_dc5c06b0c4e1","name":"CITBench","oneLine":"CITBench evaluates LLMs on interactive tabular data processing, covering table matching, cleaning, augmentation, and transformation across 18 task types and 1,296 instances.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.DB"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00018","pdf":"https://arxiv.org/pdf/2608.00018","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00018"},"evidence":{"snippet":"To bridge this gap, we introduce CITBench, a comprehensive benchmark for evaluating LLMs on interactive tabular data processing.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00018"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CITBench evaluates LLMs on interactive tabular data processing, covering table matching, cleaning, augmentation, and transformation across 18 task types and 1,296 instances.","whyItMatters":"Tabular data processing benchmarks often focus on single-turn reasoning; interactive multi-turn settings remain underevaluated.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"855d9a5c1cf28d5cb968b2fc8ab2cfffae0647e1632ccd92838e5cc0fcde9a79"},"motivation":"Tabular data processing is central to data work, and LLM-based assistants have recently shown promising capabilities in supporting such tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00018","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_carebench_48d01c35","familyId":"bmf_1e30cd79e194","name":"CAREBench","oneLine":"CAREBench evaluates language models on upstream child-safety risks with 500 prompts across twelve categories, assessing recognition, refusal, de-escalation, and redirection.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29685","pdf":"https://arxiv.org/pdf/2606.29685","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.29685"},"evidence":{"snippet":"We introduce CAREBench (Child AI Risk Evaluation), a benchmark to assess such upstream child-safety risks in language models.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29685"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"CAREBench evaluates language models on upstream child-safety risks with 500 prompts across twelve categories, assessing recognition, refusal, de-escalation, and redirection.","whyItMatters":"Child-safety evaluation often focuses on explicit material; CAREBench targets earlier risk scenarios, which could help developers identify policy gaps.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a14405e7b74c365e3e3e5754e9b0c440e62e0b07f49a2b27bf41cea2974b3b9d"},"motivation":"How can we evaluate whether frontier AI systems recognize child-safety risks before they escalate into explicit harm?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.29685","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_vg-guibench_23eefff5","familyId":"bmf_42e90ac524ad","name":"VG-GUIBench","oneLine":"VG-GUI-Bench evaluates MLLM-based GUI agents on following video tutorials to complete interactive tasks, with 1,000 long-horizon test cases and four metrics.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29445","pdf":"https://arxiv.org/pdf/2606.29445","project":"https://vg-gui-tasker.github.io/","code":"https://github.com/VG-GUI-TASKER/VG-GUI-TASKER","data":null,"hfPaper":"https://huggingface.co/papers/2606.29445"},"evidence":{"snippet":"To address this gap, we introduce VG-GUIBench (Video-Guided GUI Benchmark), a new benchmark designed to evaluate whether MLLM-based GUI agents can follow video tutorials to complete corresponding GUI interactive tasks.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":28,"hfDailySubmittedAt":null,"githubStars":19,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29445"},"ranking":{"90d":{"score":68,"rank":84,"coverage":0.7,"confidence":"Medium"}},"description":"VG-GUI-Bench evaluates MLLM-based GUI agents on following video tutorials to complete interactive tasks, with 1,000 long-horizon test cases and four metrics.","whyItMatters":"The benchmark addresses video-guided agentic tasks, complementing VideoQA benchmarks for procedural knowledge transfer.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"64a241c8ce1d20bf21e270316b808a1e681e5fe8998252866a35ceb5d8da392d"},"motivation":"Video understanding is a fundamental capability for multimodal intelligence, and recent Multimodal Large Language Models (MLLMs) have achieved remarkable performance on Video Question Answering (VideoQA) benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026","evidence":"Accepted by ECCV 2026. Project Page: https://vg-gui-tasker.github.io/","evidenceUrl":"https://arxiv.org/abs/2606.29445","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026","reviewStatus":"accepted","decisionRaw":"Accepted by ECCV 2026. Project Page: https://vg-gui-tasker.github.io/","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.29445","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by ECCV 2026. Project Page: https://vg-gui-tasker.github.io/","level":"author-claim"}]}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_the-complexity-ceiling-benchmark_2e65280a","familyId":"bmf_b80ea254c0bc","name":"The Complexity Ceiling Benchmark","oneLine":"Complexity Ceiling Benchmark evaluates sequential reasoning decay with depth scaling across grounded spatial state-tracking, symbolic pointer manipulation, and transitive relational inference.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29278","pdf":"https://arxiv.org/pdf/2606.29278","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.29278"},"evidence":{"snippet":"We introduce the Complexity Ceiling Benchmark (CCB), a controlled evaluation of how language-model reasoning decays as the number of required sequential steps grows.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29278"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Complexity Ceiling Benchmark evaluates sequential reasoning decay with depth scaling across grounded spatial state-tracking, symbolic pointer manipulation, and transitive relational inference.","whyItMatters":"It quantifies reasoning degradation with step count, which could inform model development for long-horizon tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a5549ee71f8de4a2195c0119b70c1f425d13303514a26dffb6036ddd896a7fb4"},"motivation":"We introduce the Complexity Ceiling Benchmark (CCB), a controlled evaluation of how language-model reasoning decays as the number of required sequential steps grows.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"1st Workshop on Combining Theory and Benchmarks (CTB), CTB@ICML 2026","evidence":"12 pages, 6 figures. Accepted to the 1st Workshop on Combining Theory and Benchmarks (CTB), CTB@ICML 2026","evidenceUrl":"https://arxiv.org/abs/2606.29278","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"1st Workshop on Combining Theory and Benchmarks (CTB), CTB@ICML 2026","reviewStatus":"accepted","decisionRaw":"12 pages, 6 figures. Accepted to the 1st Workshop on Combining Theory and Benchmarks (CTB), CTB@ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.29278","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"12 pages, 6 figures. Accepted to the 1st Workshop on Combining Theory and Benchmarks (CTB), CTB@ICML 2026","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_surgvla-bench_a2867ef6","familyId":"bmf_71013fdc6c32","name":"SurgVLA-Bench","oneLine":"Evaluates vision-language-action models in laparoscopic surgical robotics across 8 tasks (atomic, conditional, composite) on the SurRoL simulator, using action accuracy and semantic consistency metrics.","area":"Multimodal","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29247","pdf":"https://arxiv.org/pdf/2606.29247","project":null,"code":"https://github.com/VCL-HNU/SurgVLA","data":null,"hfPaper":"https://huggingface.co/papers/2606.29247"},"evidence":{"snippet":"To address this limitation, we present SurgVLA-Bench, the first comprehensive benchmark for evaluating VLA models in laparoscopic surgical robotics.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29247"},"ranking":{"90d":{"score":53,"rank":176,"coverage":0.55,"confidence":"Low"}},"description":"Evaluates vision-language-action models in laparoscopic surgical robotics across 8 tasks (atomic, conditional, composite) on the SurRoL simulator, using action accuracy and semantic consistency metrics.","whyItMatters":"Fills the lack of standardized surgical VLA benchmarks, enabling comparison of autoregressive vs. flow-matching models and identifying physical bottlenecks like limited field of view.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b6a15db73b403a53c4833e5b37e9415d9420c71467e1d7c56a8dd26d430a164f"},"motivation":"Vision-Language-Action (VLA) models represent a promising direction for embodied intelligence in surgical robotics.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.29247","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"VCL-HNU","organizationType":"academic-lab","sourceUrl":"https://github.com/VCL-HNU/SurgVLA","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_stemgym_d150c717","familyId":"bmf_14f397cd3d3e","name":"STEMGym","oneLine":"STEMGym is an open-source Gymnasium benchmark of 15 physics-simulated STEM worlds across five materials and four characterization tasks, scored by DEC-AUC.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29592","pdf":"https://arxiv.org/pdf/2606.29592","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.29592"},"evidence":{"snippet":"We introduce STEMGym, an open-source Gymnasium benchmark of 15 physics-simulated STEM worlds spanning five materials, three difficulty levels, and four characterisation tasks, scored by the Dose-Efficiency Curve area (DEC-AUC), a single scalar capturing the information-vs-dose Pareto frontier.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29592"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"STEMGym is an open-source Gymnasium benchmark of 15 physics-simulated STEM worlds across five materials and four characterization tasks, scored by DEC-AUC.","whyItMatters":"Addresses the need to evaluate perception, navigation, and planning trade-offs in autonomous electron microscopy under dose budgets.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a638a75b98ea1f772fe83faaaa67121f945ac07dd317a823db3270f6bc953412"},"motivation":"A central premise of autonomous scientific imaging is that smarter navigation, whether Bayesian, RL-based, or otherwise adaptive, is the principal lever for sample-efficient acquisition.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.29592","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_osworld-2-0_ee079e02","familyId":"bmf_f1ccc6d9d057","name":"OSWorld 2.0","oneLine":"Evaluates computer-use agents on 108 long-horizon real-world workflows across everyday and professional tasks, scored by binary completion at 500 steps and partial scores.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.29537","pdf":"https://arxiv.org/pdf/2606.29537","project":null,"code":"https://github.com/xlang-ai/OSWorld-V2","data":null,"hfPaper":"https://huggingface.co/papers/2606.29537"},"evidence":{"snippet":"We introduce OSWorld 2.0, a benchmark of 108 long-horizon computer-use workflows across everyday and professional tasks, designed to capture complex and challenging real-world phenomena.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":24,"hfDailySubmittedAt":null,"githubStars":256,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.29537"},"ranking":{"90d":{"score":82,"rank":13,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates computer-use agents on 108 long-horizon real-world workflows across everyday and professional tasks, scored by binary completion at 500 steps and partial scores.","whyItMatters":"Captures long-horizon, dynamic, and hidden-state challenges absent in prior benchmarks, revealing that agents fail on constraint tracking and mid-task information.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6a46fc7ce1c8ae35070e7444ec1e6176ab91d714e98416ae04dc584ef408e356"},"motivation":"Existing computer-use benchmarks fail to capture the realism, complexity, and long-horizon demands of real-world computer use, limiting their ability to reveal the limitations of frontier agents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.29537","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"XLang Lab","organizationType":"academic-lab","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_stancebench_98c43cc6","familyId":"bmf_8befc508f2a5","name":"StanceBench","oneLine":"StanceBench evaluates interpersonal stance in conversational speech across 9 dimensions using LLM-as-a-judge on the Seamless Interaction corpus.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.22658","pdf":"https://arxiv.org/pdf/2607.22658","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.22658"},"evidence":{"snippet":"We introduce StanceBench, a benchmark for measuring interpersonal stance in conversational speech and evaluating audio-capable LLMs as automated judges.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.22658"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"StanceBench evaluates interpersonal stance in conversational speech across 9 dimensions using LLM-as-a-judge on the Seamless Interaction corpus.","whyItMatters":"Addresses the gap in benchmarks for prosody and interactional nuance in speech-to-speech models, focusing on judge bias and robustness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e307dc5489a394a269ec99c7fe7f269e93a311562eb3aec2ba23c4f7274e622c"},"motivation":"Speech-to-speech dialogue models increasingly depend on prosody and interactional nuance to convey social intent, yet benchmarks for these cues remain limited.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"Interspeech 2026","evidence":"Accepted to Interspeech 2026","evidenceUrl":"https://arxiv.org/abs/2607.22658","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"Interspeech 2026","reviewStatus":"accepted","decisionRaw":"Accepted to Interspeech 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.22658","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to Interspeech 2026","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_video-mme-logical_db483a80","familyId":"bmf_406a86346e24","name":"Video-MME-Logical","oneLine":"Evaluates video temporal-logical reasoning in multimodal LLMs across 25 fine-grained task categories, with difficulty-controlled final-answer scoring and intermediate-state diagnostics.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.27828","pdf":"https://arxiv.org/pdf/2606.27828","project":null,"code":"https://github.com/Mrakas/video-mme-logical","data":null,"hfPaper":"https://huggingface.co/papers/2606.27828"},"evidence":{"snippet":"To isolate this capability, we introduce Video-MME-Logical, a controlled benchmark organized around five temporal-logical operations: state tracking, sequential counting, temporal ordering, dynamic spatiality, and structural composition.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":25,"hfDailySubmittedAt":null,"githubStars":10,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.27828"},"ranking":{"90d":{"score":64,"rank":108,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates video temporal-logical reasoning in multimodal LLMs across 25 fine-grained task categories, with difficulty-controlled final-answer scoring and intermediate-state diagnostics.","whyItMatters":"Isolates temporal-logical capabilities from static recognition, revealing significant human-model gaps and providing a scalable testbed for analysis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c86aa692865b502b97e68aab95e557501c9e79f7922ecba42d53b13fb9dae09f"},"motivation":"Recent interest in multimodal large language models (MLLMs) raises a central question: can they reason over dynamic visual evidence rather than merely recognize objects or events in individual frames?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.27828","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Mrakas","organizationType":"academic-lab","sourceUrl":"https://github.com/Mrakas/video-mme-logical","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_tua-bench_56e7e977","familyId":"bmf_6458a26e1270","name":"TUA-Bench","oneLine":"Evaluates terminal-use agents on 120 real-world tasks across five families (document editing, email, web info, scientific/engineering workflows) using execution-based scoring.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.28480","pdf":"https://arxiv.org/pdf/2606.28480","project":"https://www.tuabench.ai","code":"https://github.com/facebookresearch/TUA-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.28480"},"evidence":{"snippet":"We introduce TUA-Bench, a general-purpose benchmark for terminal-use agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":48,"hfDailySubmittedAt":null,"githubStars":46,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.28480"},"ranking":{"90d":{"score":76,"rank":37,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates terminal-use agents on 120 real-world tasks across five families (document editing, email, web info, scientific/engineering workflows) using execution-based scoring.","whyItMatters":"Provides a broad, realistic terminal benchmark beyond coding, showing frontier agents achieve only 65.8% success, highlighting gaps in general-purpose digital work.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6b4c90791bb32c4e309ff9509a785f3211b5bd10d48a36c6d542ace7f7d1455d"},"motivation":"As large language models and harness frameworks continue to advance, agents operating in terminals are increasingly capable of performing a broader range of general computer-use tasks beyond coding.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.28480","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Facebook Research","organizationType":"company-research-lab","sourceUrl":"https://github.com/facebookresearch/TUA-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_spatialuav_6900b974","familyId":"bmf_e9cbb4f3f030","name":"SpatialUAV","oneLine":"Evaluates spatial intelligence in low-altitude UAV scenarios across 14 task types covering semantic discrimination, spatial relations, collaboration, and motion understanding, with 7 input configurations and 9 answer formats.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.27876","pdf":"https://arxiv.org/pdf/2606.27876","project":null,"code":"https://github.com/Hyu-Zhang/SpatialUAV","data":null,"hfPaper":"https://huggingface.co/papers/2606.27876"},"evidence":{"snippet":"To address these gaps, we introduce SpatialUAV, a real low-altitude UAV benchmark comprising 4,331 curated instances across 14 fine-grained task types, covering semantic discrimination, spatial relation, aerial--aerial collaboration, aerial--ground collaboration, and motion understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.27876"},"ranking":{"90d":{"score":47,"rank":215,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates spatial intelligence in low-altitude UAV scenarios across 14 task types covering semantic discrimination, spatial relations, collaboration, and motion understanding, with 7 input configurations and 9 answer formats.","whyItMatters":"Targets under-evaluated 3D spatial and multi-view reasoning in UAV benchmarks, revealing bottlenecks in cross-view association and geometric reasoning for current VLMs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"de5b56c74e975bed6465829bbf0b94bf4b993a6dc6e85d808af5a576fe45b083"},"motivation":"Spatial intelligence is essential for low-altitude unmanned aerial vehicle (UAV) perception, collaboration, and navigation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.27876","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Hyu-Zhang","organizationType":"academic-lab","sourceUrl":"https://github.com/Hyu-Zhang/SpatialUAV","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_robustmad_0c7403de","familyId":"bmf_191732065a96","name":"RobustMAD","oneLine":"Evaluates robustness of multimodal small language models for industrial anomaly detection across open-ended queries and visual degradations, using multiple-choice accuracy and LLM-judged open-ended responses.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Robustness"],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.16243","pdf":"https://arxiv.org/pdf/2607.16243","project":"https://openreview.net/forum?id=skrA9UYNIZ","code":"https://github.com/en-research/RobustMAD","data":null,"hfPaper":"https://huggingface.co/papers/2607.16243"},"evidence":{"snippet":"To address this gap, we develop RobustMAD, the first deployment-motivated benchmark, designed to comprehensively evaluate model robustness through diverse open-ended queries spanning object understanding, anomaly detection, unanswerable problems, and visual quality degradations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.16243"},"ranking":{"90d":{"score":53,"rank":178,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates robustness of multimodal small language models for industrial anomaly detection across open-ended queries and visual degradations, using multiple-choice accuracy and LLM-judged open-ended responses.","whyItMatters":"Assesses deployability of compact models in real-world industrial conditions, identifying failure modes like fragile grounding and hallucination on ill-posed queries.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6ea21137d1a73d54364f7629546ff74194bead95c73b6f9b122de37c977e15d3"},"motivation":"Multimodal industrial anomaly inspection assistants are a critical component of next-generation smart factories, enabling interactive vision-language-based querying.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"publication in Transactions on Machine Learning Research (TMLR), 2026 at https://openreview","evidence":"Accepted for publication in Transactions on Machine Learning Research (TMLR), 2026 at https://openreview.net/forum?id=skrA9UYNIZ","evidenceUrl":"https://arxiv.org/abs/2607.16243","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"publication in Transactions on Machine Learning Research (TMLR), 2026 at https://openreview","reviewStatus":"accepted","decisionRaw":"Accepted for publication in Transactions on Machine Learning Research (TMLR), 2026 at https://openreview.net/forum?id=skrA9UYNIZ","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2607.16243","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted for publication in Transactions on Machine Learning Research (TMLR), 2026 at https://openreview.net/forum?id=skrA9UYNIZ","level":"author-claim"}]}],"publishers":[{"name":"EN Research","organizationType":"academic-lab","sourceUrl":"https://github.com/en-research/RobustMAD","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_normact_ade65389","familyId":"bmf_a47d8a75095d","name":"NormAct","oneLine":"NormAct evaluates embodied agents on 550 TongSim scenarios where the same goal permits norm-compliant or norm-violating action sequences, testing whether agents infer and apply scene-relevant social norms during ordinary tasks.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.27826","pdf":"https://arxiv.org/pdf/2606.27826","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.27826"},"evidence":{"snippet":"We introduce NormAct, a benchmark of 550 TongSim scenarios in which the same goal permits norm-compliant or norm-violating action sequences.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.27826"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"NormAct evaluates embodied agents on 550 TongSim scenarios where the same goal permits norm-compliant or norm-violating action sequences, testing whether agents infer and apply scene-relevant social norms during ordinary tasks.","whyItMatters":"NormAct addresses the gap between goal achievement and proactive norm compliance, providing a way to assess whether embodied agents respect unstated social norms without prompting, which is critical for real-world deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8e179a3fb04d0764d24772bf6ddb92b34e4c78779e77270cb417dba995d152ab"},"motivation":"Embodied agents driven by multimodal large language models (MLLMs) can often complete everyday tasks from visual observations, but goal achievement does not establish whether they proactively respect unstated social norms.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.27826","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_imcbench_04071d33","familyId":"bmf_420ea5d66c7d","name":"IMCBench","oneLine":"IMCBench evaluates multimodal LLMs in image-grounded, multi-turn medical conversations, scoring safety, accuracy, and uncertainty use on a 1-5 scale.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.28556","pdf":"https://arxiv.org/pdf/2606.28556","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.28556"},"evidence":{"snippet":"To address this gap, we introduce IMCBench, an image-grounded, multi-turn medical conversation benchmark that pairs real, publicly available clinical images with synthetic patient profiles to simulate realistic patient-clinician interactions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.28556"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"IMCBench evaluates multimodal LLMs in image-grounded, multi-turn medical conversations, scoring safety, accuracy, and uncertainty use on a 1-5 scale.","whyItMatters":"Addresses the gap in medical AI benchmarks by combining clinical images with multi-turn dialogue, enabling assessment of diagnostic accuracy alongside patient safety and uncertainty management.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2fef12f4c3f66365e3506c1fbdcd60c65c5e5fa36c7811ad5d18a4b1aa727fda"},"motivation":"Recent advances in large language models and vision-language models have enabled reasoning over multimodal data, offering opportunities for clinical applications such as decision support and triaging.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECML PKDD 2026","evidence":"Accepted at ECML PKDD 2026. 22 pages, 2 figures","evidenceUrl":"https://arxiv.org/abs/2606.28556","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECML PKDD 2026","reviewStatus":"accepted","decisionRaw":"Accepted at ECML PKDD 2026. 22 pages, 2 figures","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.28556","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at ECML PKDD 2026. 22 pages, 2 figures","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_humanmovevqa_d7611820","familyId":"bmf_21e6a36c45a9","name":"HumanMoveVQA","oneLine":"HumanMoveVQA evaluates video MLLMs on reasoning about human trajectory and orientation changes in videos, using a first-frame anchored world coordinate system and 10K question-answer pairs across seven reasoning categories.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.27999","pdf":"https://arxiv.org/pdf/2606.27999","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.27999"},"evidence":{"snippet":"We introduce HumanMoveVQA, the first comprehensive benchmark designed to evaluate global trajectory and orientation reasoning from an exocentric perspective.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.27999"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HumanMoveVQA evaluates video MLLMs on reasoning about human trajectory and orientation changes in videos, using a first-frame anchored world coordinate system and 10K question-answer pairs across seven reasoning categories.","whyItMatters":"Existing benchmarks fail to probe global human motion in space over time; HumanMoveVQA targets this gap, but without accessible data or code its practical value for model comparison remains unclear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5bf1227adaba81623a969cd0b3dc8ad8fdb45c08f7403f652ddcd2b4b724b9fa"},"motivation":"Despite the rapid advance of Multimodal Large Language Models (MLLMs) in high-level video understanding, a fundamental bottleneck remains: these models collapse complex human motion into coarse semantic labels.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.27999","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_gptnt_3088efa1","familyId":"bmf_fa478faf966d","name":"GPTNT","oneLine":"GPTNT evaluates multimodal agents on real-time collaborative bomb defusal in the game Keep Talking and Nobody Explodes, requiring asynchronous communication under time pressure and information asymmetry. Success is measured by defusing procedurally generated bombs.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-06-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.28514","pdf":"https://arxiv.org/pdf/2606.28514","project":"https://gptnt.github.io","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.28514"},"evidence":{"snippet":"We introduce GPTNT, a benchmark built on the cooperative video game Keep Talking and Nobody Explodes, in which two agents must coordinate to defuse procedurally generated bomb puzzles against a live countdown.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.28514"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"GPTNT evaluates multimodal agents on real-time collaborative bomb defusal in the game Keep Talking and Nobody Explodes, requiring asynchronous communication under time pressure and information asymmetry. Success is measured by defusing procedurally generated bombs.","whyItMatters":"Current benchmarks isolate collaboration components; GPTNT captures time pressure, information asymmetry, and imperfect communication together, providing a realistic test for multimodal systems that current evaluations leave unmeasured.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9b8051d26fd905dc78c04a1d4c48f0f4e90b709ad10ee25058c269b4eb05572b"},"motivation":"Multimodal models are increasingly deployed to solve tasks collaboratively with humans or other artificial agents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.28514","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GPTNT Project","organizationType":"academic-lab","sourceUrl":"https://gptnt.github.io","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_discobench_0216743e","familyId":"bmf_d21d9bcb5de7","name":"DiscoBench","oneLine":"DiscoBench evaluates search agents on clarification-aware deep search, covering 211 samples and 463 ambiguity instances across 11 domains, with four ambiguity types, measuring task utility, ambiguity detection, interaction strategy, and cost efficiency.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.27669","pdf":"https://arxiv.org/pdf/2606.27669","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.27669"},"evidence":{"snippet":"To address this gap, we introduce DiscoBench, a benchmark for clarification-aware deep search, designed to evaluate whether search agents can proactively identify ambiguity, ask effective clarification questions, and recover correct reasoning paths through user interaction.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":16,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.27669"},"ranking":{"90d":{"score":54,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"DiscoBench evaluates search agents on clarification-aware deep search, covering 211 samples and 463 ambiguity instances across 11 domains, with four ambiguity types, measuring task utility, ambiguity detection, interaction strategy, and cost efficiency.","whyItMatters":"This benchmark addresses the gap in evaluating search agents' ability to handle ambiguous and underspecified queries, which is common in real-world search. It assesses proactive clarification and interaction efficiency, offering practical value for improving agent decision-making in complex information-seeking tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f92a3e3484e5f9ff9bdafbe536e98d73eeef3d8c92884d2c801a9fa46840e3fd"},"motivation":"Search agents powered by large language models (LLMs) are increasingly used to solve complex information-seeking tasks, requiring multi-step retrieval and reasoning to fulfill user goals.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.27669","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_animation2code_7763a440","familyId":"bmf_0c37430e63cf","name":"Animation2Code","oneLine":"Animation2Code evaluates temporal visual reasoning in video-to-code generation. It includes 1,069 web animation videos with corresponding HTML/CSS/JavaScript implementations, and uses appearance and temporal similarity metrics to assess model performance.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Reasoning","Code generation"],"topics":["Code","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.28593","pdf":"https://arxiv.org/pdf/2606.28593","project":"https://anya-ji.github.io/animation2code-website","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.28593"},"evidence":{"snippet":"To this end, we introduce Animation2Code, a benchmark for evaluating temporal visual reasoning via reconstructing executable web animation code from videos.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.28593"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Animation2Code evaluates temporal visual reasoning in video-to-code generation. It includes 1,069 web animation videos with corresponding HTML/CSS/JavaScript implementations, and uses appearance and temporal similarity metrics to assess model performance.","whyItMatters":"Animation2Code addresses the lack of benchmarks for temporal dynamics in visual-to-code tasks. It provides a way to measure both visual fidelity and temporal alignment, which is crucial for applications requiring precise animation reconstruction.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"da545a948dca7da7b5edbe95ef4c193b1c43de883ab31dce816a59a670930114"},"motivation":"While recent vision-language models (VLMs) have achieved significant improvements on static visual-to-code tasks such as generating code for webpages, charts, or SVGs, it remains unclear whether they can recover temporal dynamics when motion is present.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.28593","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Animation2Code Team","organizationType":"academic-lab","sourceUrl":"https://anya-ji.github.io/animation2code-website","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_airgroundbench_74494075","familyId":"bmf_10bc9f42a87c","name":"AirGroundBench","oneLine":"AirGroundBench evaluates multi-view spatial intelligence in multimodal large language models through UAV-UGV collaborative tasks, including 62,000 dual-view multiple-choice questions and 115 navigation episodes across 11 simulated environments.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.28049","pdf":"https://arxiv.org/pdf/2606.28049","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.28049"},"evidence":{"snippet":"We present AirGroundBench, a diagnostic benchmark for evaluating multi-view spatial intelligence in heterogeneous UAV-UGV collaboration.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.28049"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AirGroundBench evaluates multi-view spatial intelligence in multimodal large language models through UAV-UGV collaborative tasks, including 62,000 dual-view multiple-choice questions and 115 navigation episodes across 11 simulated environments.","whyItMatters":"This benchmark addresses the gap in assessing geometric consistency across heterogeneous views, providing a structured evaluation for capabilities like cross-view alignment and spatial reasoning that are critical for embodied decision-making.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"67a9fbeb96f546c48238838d660a8aa60cacac7fe43d51f51e9a6716b358389e"},"motivation":"In recent years, multimodal large language models (MLLMs) have shown strong potential for embodied intelligence, yet their ability to maintain geometrically consistent spatial understanding across heterogeneous views remains under-evaluated.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.28049","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AirGroundBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.28049","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_unison_ffbd3e3c","familyId":"bmf_b3704083f1b6","name":"Unison","oneLine":"Evaluates unified multimodal models on joint understanding and generation across four dimensions: internal consistency, understanding-guided generation, generation-guided understanding, and mutual enhancement, using 2,169 task samples.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26984","pdf":"https://arxiv.org/pdf/2606.26984","project":null,"code":"https://github.com/FudanCVL/Unison","data":null,"hfPaper":"https://huggingface.co/papers/2606.26984"},"evidence":{"snippet":"To bridge this gap, we introduce Unison, a comprehensive benchmark comprising 2,169 high-quality unified task samples, designed to evaluate joint understanding and generation in unified multimodal models.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":14,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26984"},"ranking":{"90d":{"score":61,"rank":128,"coverage":0.55,"confidence":"Low"}},"description":"Evaluates unified multimodal models on joint understanding and generation across four dimensions: internal consistency, understanding-guided generation, generation-guided understanding, and mutual enhancement, using 2,169 task samples.","whyItMatters":"Existing benchmarks assess understanding and generation in isolation, missing the synergy between them. This benchmark provides a diagnostic and human-aligned evaluation to measure and compare integrated capabilities, aiding model development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"702c52da8df55316ee62da579bb57a5752fab832cf84ff1bbd8436c3b79ec1e0"},"motivation":"Unified multimodal models capable of both understanding and generation have achieved remarkable strides.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.26984","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FudanCVL","organizationType":"academic-lab","sourceUrl":"https://github.com/FudanCVL/Unison","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_socialpersona_4370a8a3","familyId":"bmf_6f5f69e2ce5e","name":"SocialPersona","oneLine":"SocialPersona evaluates multimodal LLMs on recovering revealed preferences from longitudinal social-media timelines and using them in dialogue. It includes 171 user timelines, 2,597 human-verified preference tags across seven domains, and supports profile construction and response generation tasks.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26654","pdf":"https://arxiv.org/pdf/2606.26654","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.26654"},"evidence":{"snippet":"We introduce SocialPersona, a benchmark for evaluating whether multimodal large language models (MLLMs) can recover revealed preferences from longitudinal social-media timelines and use them in dialogue.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26654"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SocialPersona evaluates multimodal LLMs on recovering revealed preferences from longitudinal social-media timelines and using them in dialogue. It includes 171 user timelines, 2,597 human-verified preference tags across seven domains, and supports profile construction and response generation tasks.","whyItMatters":"Existing personalization benchmarks rely on explicitly stated preferences; SocialPersona tests inference from natural multimodal traces, a harder capability. It provides a reusable benchmark for measuring progress on long-horizon user modeling and personalized dialogue.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"47008b2a46569b0f3b0a4421f92d644c779fa1404b9c51fb27ca0fba3eb377bb"},"motivation":"Personalized language-model assistants are often evaluated through a memory lens: can a model recall preferences users have explicitly stated in dialogue?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.26654","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_scbench-long_49d2490a","familyId":"bmf_8fb55cb2c14b","name":"scBench-Long","oneLine":"scBench-Long evaluates long-horizon single-cell biology reasoning. Agents must recover scientific conclusions from raw or near-raw data without prescribed methods. It contains 21 evaluations spanning diverse biological contexts, with deterministic grading and trajectory rubrics.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["q-bio.GN"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26563","pdf":"https://arxiv.org/pdf/2606.26563","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.26563"},"evidence":{"snippet":"We introduce scBench-Long, a benchmark for long-horizon single-cell biology in which agents must recover scientific conclusions from raw or near-raw data without prescribed methods.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26563"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"scBench-Long evaluates long-horizon single-cell biology reasoning. Agents must recover scientific conclusions from raw or near-raw data without prescribed methods. It contains 21 evaluations spanning diverse biological contexts, with deterministic grading and trajectory rubrics.","whyItMatters":"Existing AI-biology benchmarks measure broad knowledge or local steps; scBench-Long assesses end-to-end scientific claim production. It provides a reusable evaluation for long-horizon reasoning in single-cell data analysis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0ce8128d983bac74942c4aee784ff35a4f35064bc94ddcb88f4b278bcc660cb9"},"motivation":"Single-cell studies require analysts to convert raw measurements into specific biological claims through multi-step workflows and integration of metadata, assay context, and auxiliary evidence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.26563","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_redvox_a5dbb799","familyId":"bmf_ecfd25a09758","name":"RedVox","oneLine":"RedVox is a multilingual safety and fairness benchmark for speech models, built on real voices. It covers unsafe and unfair stereotypical requests across five languages and evaluates models under naturalistic conditions.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26968","pdf":"https://arxiv.org/pdf/2606.26968","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.26968"},"evidence":{"snippet":"To address this gap, we introduce RedVox, a multilingual safety and fairness benchmark for audio and speech built on real voices, covering unsafe and unfair stereotypical requests across five languages (English, French, Italian, Spanish, and German).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":16,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26968"},"ranking":{"90d":{"score":54,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"RedVox is a multilingual safety and fairness benchmark for speech models, built on real voices. It covers unsafe and unfair stereotypical requests across five languages and evaluates models under naturalistic conditions.","whyItMatters":"Speech models are deployed globally but safety evaluations are mostly English-only. RedVox provides a reusable benchmark to assess cross-lingual safety gaps and the impact of spoken input.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f54c855deb4269689c5e24c7e169df147ce34cb88d117233c7c7e6278bc0eb26"},"motivation":"Speech-capable models are increasingly deployed in real-world applications across languages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.26968","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_phyeditbench_2e24a281","familyId":"bmf_5eed3359c9a8","name":"PhyEditBench","oneLine":"PhyEditBench evaluates physics-aware image editing. It includes 238 real-world instances and 35 synthetic anti-physics instances across 12 physical subclasses, with VLM-based scoring on consistency, instruction following, physical plausibility, and image quality.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26551","pdf":"https://arxiv.org/pdf/2606.26551","project":null,"code":"https://github.com/Previsior/PhyEditBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.26551"},"evidence":{"snippet":"To address this, we introduce PhyEditBench, a benchmark designed to assess the physical understanding of editing models.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26551"},"ranking":{"90d":{"score":45,"rank":233,"coverage":0.7,"confidence":"Medium"}},"description":"PhyEditBench evaluates physics-aware image editing. It includes 238 real-world instances and 35 synthetic anti-physics instances across 12 physical subclasses, with VLM-based scoring on consistency, instruction following, physical plausibility, and image quality.","whyItMatters":"Image editing benchmarks often overlook physics reasoning; PhyEditBench provides a reusable evaluation to measure physical coherence in edited outputs, important for real-world applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"33de1c095085029be9572bd32768609ba31aa1377a2b22e8cf95778d340dadc1"},"motivation":"While instruction-based image editing, enabled by multi-modal generative models, has advanced significantly, existing benchmarks lack a comprehensive evaluation of physics-based reasoning, a critical capability for handling real-world scenarios.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026","evidence":"19 pages, 6 figures, 2 tables. Accepted to ECCV 2026","evidenceUrl":"https://arxiv.org/abs/2606.26551","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026","reviewStatus":"accepted","decisionRaw":"19 pages, 6 figures, 2 tables. Accepted to ECCV 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.26551","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"19 pages, 6 figures, 2 tables. Accepted to ECCV 2026","level":"author-claim"}]}],"publishers":[{"name":"Previsior","organizationType":"academic-lab","sourceUrl":"https://github.com/Previsior/PhyEditBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_omniref-bench_79744dc3","familyId":"bmf_0e055f109041","name":"OmniRef-Bench","oneLine":"OmniRef-Bench is introduced in the paper but primarily serves to evaluate the proposed DyRef method. It is not presented as a standalone public benchmark.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26947","pdf":"https://arxiv.org/pdf/2606.26947","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.26947"},"evidence":{"snippet":"To better assess model performance on complex MRIG tasks, we introduce OmniRef-Bench, a benchmark that covers complex combinations of reference image types and a large number of reference images.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26947"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OmniRef-Bench is introduced in the paper but primarily serves to evaluate the proposed DyRef method. It is not presented as a standalone public benchmark.","whyItMatters":"N/A","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3a3feb743ca8bdc00dc0bacc0e3fe35416491a5318e8eb8f7cc3b3882e1526ca"},"motivation":"While personalized image generation has achieved remarkable progress, multi-reference image generation (MRIG) remains a challenging task.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV2026","evidence":"Accepted by ECCV2026","evidenceUrl":"https://arxiv.org/abs/2606.26947","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV2026","reviewStatus":"accepted","decisionRaw":"Accepted by ECCV2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.26947","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by ECCV2026","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_nuclearqav2_79073391","familyId":"bmf_30d04771464d","name":"NuclearQAv2","oneLine":"NuclearQAv2 evaluates LLMs on nuclear engineering knowledge with approximately 1,240 QA pairs covering boolean, numeric, and verbal questions. It uses structured prompting for automated question generation and response evaluation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.27047","pdf":"https://arxiv.org/pdf/2606.27047","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.27047"},"evidence":{"snippet":"To address the need for systematic evaluation in this domain, we introduce NuclearQAv2, a benchmark for assessing LLMs on nuclear engineering knowledge.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.27047"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"NuclearQAv2 evaluates LLMs on nuclear engineering knowledge with approximately 1,240 QA pairs covering boolean, numeric, and verbal questions. It uses structured prompting for automated question generation and response evaluation.","whyItMatters":"Technical domains like nuclear engineering require reliable LLM evaluation. NuclearQAv2 provides a scalable benchmark to assess factual knowledge, quantitative reasoning, and conceptual understanding in this domain.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5538f6caecec97772f7e01820aab1d86b34a5dd871d9351c8dda55373eaa068b"},"motivation":"Large language models (LLMs) have demonstrated strong performance across a wide range of tasks, but ensuring their reliability in highly technical domains remains a significant challenge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.27047","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_memobench_def5688a","familyId":"bmf_e8d56608be68","name":"MemoBench","oneLine":"MemoBench evaluates world modeling in video generation via a disappear-and-reappear paradigm. It includes 360 ground-truth clips and an evaluation suite with automated metrics and VQA across four diagnostic pillars.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.27537","pdf":"https://arxiv.org/pdf/2606.27537","project":null,"code":"https://github.com/MemoBench-Team/MemoBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.27537"},"evidence":{"snippet":"To bridge this gap, we introduce MemoBench, a diagnostic benchmark built around the disappear-and-reappear paradigm in dynamically changing environments: a target object undergoes a physical process, disappears from view, and must be correctly recovered in its updated state upon reappearance.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":6,"hfDailySubmittedAt":null,"githubStars":37,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.27537"},"ranking":{"90d":{"score":70,"rank":73,"coverage":0.7,"confidence":"Medium"}},"description":"MemoBench evaluates world modeling in video generation via a disappear-and-reappear paradigm. It includes 360 ground-truth clips and an evaluation suite with automated metrics and VQA across four diagnostic pillars.","whyItMatters":"Current benchmarks assess memory consistency only when objects remain visible; MemoBench tests object recovery after occlusion in dynamic scenes, providing a reusable diagnostic for world models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bcf200e2fffb5ff18b81ee5d86bcf58665a22562b7be9ebf498b18c326981ff2"},"motivation":"Video generation models aspire to simulate dynamic environments, and several benchmarks now evaluate memory consistency across frames.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.27537","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MemoBench Team","organizationType":"community","sourceUrl":"https://github.com/MemoBench-Team/MemoBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_lcs-bench_45178465","familyId":"bmf_5c62eda733ab","name":"LCS-Bench","oneLine":"LCS-Bench is a theory-scale benchmark for auto-formalization in logics for computer science. It includes 327 textbook items, over 4,076 Lean declarations, and supports five evaluation tracks with definitional equivalence checkers.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26525","pdf":"https://arxiv.org/pdf/2606.26525","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.26525"},"evidence":{"snippet":"In this paper, we introduce LCS-Bench, a stand-alone, theory-scale benchmark based on Logics for Computer Science.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26525"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LCS-Bench is a theory-scale benchmark for auto-formalization in logics for computer science. It includes 327 textbook items, over 4,076 Lean declarations, and supports five evaluation tracks with definitional equivalence checkers.","whyItMatters":"Auto-formalization at theory scale remains challenging. LCS-Bench provides a reusable benchmark to measure consistency, faithfulness, and correctness of formalization systems, enabling progress in scalable verification.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"357a38782c0ca8707c63929c38864bbe3eafa0f5b15f6979e30fc111927b4d94"},"motivation":"Auto-formalization is critical for scalable formal verification, but existing progress largely focuses on isolated statements, while theory-scale auto-formalization, which coherently translates hundreds of interdependent definitions, lemmas, and theorems, remains open due to challenges in consistency, faithfulness, scalability, and correctness.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.26525","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_harmvideobench_cc5648e1","familyId":"bmf_7d36f4d3eff5","name":"HarmVideoBench","oneLine":"Evaluates harmful video understanding in large multimodal models using 1,379 videos and 4,137 multiple-choice questions across three hierarchical dimensions: Observable Evidence, Clip-Internal Meaning, and Beyond-Clip Reasoning.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.27187","pdf":"https://arxiv.org/pdf/2606.27187","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.27187"},"evidence":{"snippet":"To address these problems, we present HarmVideoBench, a multi-layered diagnostic benchmark comprising 1,379 videos paired with 4,137 multiple-choice questions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.27187"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates harmful video understanding in large multimodal models using 1,379 videos and 4,137 multiple-choice questions across three hierarchical dimensions: Observable Evidence, Clip-Internal Meaning, and Beyond-Clip Reasoning.","whyItMatters":"Addresses the gap of shallow binary classification in harmful video benchmarks by testing deep contextual understanding, and the absence of explanatory rationales, providing a diagnostic tool for model evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"da19fcc6a36e1ba6b9e1ab2fdf885087bfa948216d6cc4da0be732640dff2ffe"},"motivation":"Large vision-language models (LVLMs) have recently shown immense potential in automated content moderation, sparking growing interest in developing harmful-video benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.27187","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_foresightsafety-vla_a9025fe7","familyId":"bmf_f09eeaf1bfcf","name":"ForesightSafety-VLA","oneLine":"A diagnostic safety benchmark for vision-language-action models, evaluating policies across physical interaction, instruction, and perception safety with cumulative cost and risk exposure metrics.","area":"Safety & Trustworthiness","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Multimodal","Safety"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.27079","pdf":"https://arxiv.org/pdf/2606.27079","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.27079"},"evidence":{"snippet":"To address this gap, we introduce ForesightSafety-VLA, a diagnostic benchmark that makes safety the primary evaluation target for VLA systems.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.27079"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A diagnostic safety benchmark for vision-language-action models, evaluating policies across physical interaction, instruction, and perception safety with cumulative cost and risk exposure metrics.","whyItMatters":"The benchmark addresses the lack of systematic safety evaluation for embodied VLA models, offering process-level risk metrics and a taxonomy to localize failure sources, which supports model selection and safety improvements.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"91363d178a06d8cad577c721e4b94b919b1f532e69b5384ec53f38bacda3980f"},"motivation":"In embodied intelligence, safety is a prerequisite for reliable robot deployment in the physical world.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.27079","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_flamevqa_fd5c1a64","familyId":"bmf_5e31ad44d17e","name":"FlameVQA","oneLine":"FlameVQA is a multiple-choice visual question answering benchmark for UAV-based wildfire monitoring, built on FLAME 3 with paired RGB and radiometric thermal images. It includes 34 questions per image across six capability groups, covering detection, localization, coverage estimation, cross-modal reasoning, and flight planning. Labels are generated via MLLM assistance, deterministic thermal rules, and human auditing. The dataset and code are open-source.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Safety","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.27128","pdf":"https://arxiv.org/pdf/2606.27128","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.27128"},"evidence":{"snippet":"We introduce FlameVQA, a multiple-choice visual question answering benchmark for UAV-based wildfire intelligence built on FLAME 3, leveraging paired RGB imagery and radiometric thermal TIFFs for temperature-grounded, safety-critical reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.27128"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FlameVQA is a multiple-choice visual question answering benchmark for UAV-based wildfire monitoring, built on FLAME 3 with paired RGB and radiometric thermal images. It includes 34 questions per image across six capability groups, covering detection, localization, coverage estimation, cross-modal reasoning, and flight planning. Labels are generated via MLLM assistance, deterministic thermal rules, and human auditing. The dataset and code are open-source.","whyItMatters":"FlameVQA addresses the lack of benchmarks for evaluating vision-language models in safety-critical wildfire scenarios where RGB-only interpretation is insufficient. It provides a standardized evaluation for capabilities like smoke detection and coverage estimation, which are critical for practical deployment of MLLMs in disaster monitoring.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a632b6de7e18eb2936658e9aa29a3bd19fe205b989d42a1ee97d624c48417fea"},"motivation":"Wildfire monitoring from UAVs requires reliable reasoning over complex aerial scenes, where smoke, scale variation, and occlusions often limit RGB-only interpretation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.27128","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_dmv-bench_d16bc24a","familyId":"bmf_bac3db190dd7","name":"DMV-Bench","oneLine":"DMV-Bench evaluates visual memory of multimodal agents in an interactive shopping environment with 1,000 product variants. Agents run autonomous sessions and must recall cued product images via exact URL match, with text leakage controlled.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents","Multimodal"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.27499","pdf":"https://arxiv.org/pdf/2606.27499","project":null,"code":"https://github.com/yyyujintang/DMV-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.27499"},"evidence":{"snippet":"We introduce DMV-Bench (Code: https://github.com/yyyujintang/DMV-Bench), the first interactive benchmark for multimodal-agent visual memory.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.27499"},"ranking":{"90d":{"score":46,"rank":224,"coverage":0.7,"confidence":"Medium"}},"description":"DMV-Bench evaluates visual memory of multimodal agents in an interactive shopping environment with 1,000 product variants. Agents run autonomous sessions and must recall cued product images via exact URL match, with text leakage controlled.","whyItMatters":"Existing agent memory benchmarks focus on text; DMV-Bench isolates visual memory needs in interactive settings, offering a controlled protocol for comparing agent architectures on pixel-based recall across varying session lengths.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4228d80f2293441b65e01027ef3ace973887d6a3a795b65dd5354248bcd0fee9"},"motivation":"Research on agent memory has matured rapidly, but almost entirely on the text side: few existing benchmarks ask, in an interactive environment, when an agent genuinely needs to remember what it saw rather than what it could write down.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.27499","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DMV-Bench Team","organizationType":"academic-lab","sourceUrl":"https://github.com/yyyujintang/DMV-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_dicobench_558e401f","familyId":"bmf_1608a2621237","name":"DiCoBench","oneLine":"DiCoBench evaluates multimodal LLMs on multi-image fine-grained perception using high-resolution (up to 2K) image pairs. It includes 765 multiple-choice questions across two tracks: differential and commonality visual cues, covering 8 perception tasks, with exact-match scoring.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26602","pdf":"https://arxiv.org/pdf/2606.26602","project":null,"code":"https://github.com/PKU-ICST-MIPL/DICO_Bench_ECCV2026","data":null,"hfPaper":"https://huggingface.co/papers/2606.26602"},"evidence":{"snippet":"To bridge this gap, we introduce DiCoBench, a comprehensive, multi-image high-resolution benchmark designed for cross-image fine-grained perception.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26602"},"ranking":{"90d":{"score":29,"rank":381,"coverage":0.7,"confidence":"Medium"}},"description":"DiCoBench evaluates multimodal LLMs on multi-image fine-grained perception using high-resolution (up to 2K) image pairs. It includes 765 multiple-choice questions across two tracks: differential and commonality visual cues, covering 8 perception tasks, with exact-match scoring.","whyItMatters":"Existing benchmarks rely on explicit textual cues or low resolution, whereas DiCoBench targets autonomous discovery of subtle visual cues in high-resolution pairs. It provides a challenging testbed with a large human-model performance gap, aiding progress in complex multi-image understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"057c225e8c3eaf519674e7e2633fffa5561c01c21c064990aba6ff97ba03bc8e"},"motivation":"Recent advancements in Multimodal Large Language Models (MLLMs) have demonstrated impressive fine-grained perception capabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026","evidence":"Accepted by ECCV 2026. Project page with code: https://github.com/PKU-ICST-MIPL/DICO_Bench_ECCV2026","evidenceUrl":"https://arxiv.org/abs/2606.26602","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026","reviewStatus":"accepted","decisionRaw":"Accepted by ECCV 2026. Project page with code: https://github.com/PKU-ICST-MIPL/DICO_Bench_ECCV2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.26602","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by ECCV 2026. Project page with code: https://github.com/PKU-ICST-MIPL/DICO_Bench_ECCV2026","level":"author-claim"}]}],"publishers":[{"name":"PKU-ICST-MIPL","organizationType":"academic-lab","sourceUrl":"https://github.com/PKU-ICST-MIPL/DICO_Bench_ECCV2026","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_watchact_107de5f7","familyId":"bmf_84d3a4a518a6","name":"WatchAct","oneLine":"WatchAct evaluates robot manipulation from observed human behavior. Each instance pairs a human-action video and a language instruction with an aligned simulator scene and an executable LIBERO task. It covers 3,000 long-horizon instances across 14 tasks in four capability domains: Event Grounding, Procedural Reasoning, Implicit Intent Inference, and Episodic Reasoning. The evaluation protocol separately measures video-to-plan reasoning, policy execution under oracle plans, and full task completion, in simulation and on a Franka Research 3 robot.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26443","pdf":"https://arxiv.org/pdf/2606.26443","project":"https://baiqi-li.github.io/watchact_page/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.26443"},"evidence":{"snippet":"We introduce WatchAct, a benchmark for robot manipulation grounded in observed human behavior.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26443"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"WatchAct evaluates robot manipulation from observed human behavior. Each instance pairs a human-action video and a language instruction with an aligned simulator scene and an executable LIBERO task. It covers 3,000 long-horizon instances across 14 tasks in four capability domains: Event Grounding, Procedural Reasoning, Implicit Intent Inference, and Episodic Reasoning. The evaluation protocol separately measures video-to-plan reasoning, policy execution under oracle plans, and full task completion, in simulation and on a Franka Research 3 robot.","whyItMatters":"Existing manipulation benchmarks typically evaluate from a single current image, lacking grounding in observed human behavior. WatchAct fills this gap by assessing robots' ability to reason about events, procedures, intents, and scene changes from video, which is critical for real-world human-robot collaboration. It provides a disentangled evaluation to isolate reasoning and execution failures, offering practical value for developing and comparing systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"19fa056d94a3b2468d06aa862d3835c64ee43c1fa5bb28ac89328e2ec606faa3"},"motivation":"A robot working alongside people must reason about what they have done, in what order, and with what intent.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.26443","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_triviewbench_b0684034","familyId":"bmf_75312dbd377d","name":"TriViewBench","oneLine":"TriViewBench evaluates multimodal LLMs on multi-view structural reasoning using synthetic 3D scenes with controlled object count and occlusion. It comprises 1,923 scenes and over 14,000 QA pairs across four complexity levels and three reasoning categories: Local Decision, Object Counting, and Global Recovery.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Geometric reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26029","pdf":"https://arxiv.org/pdf/2606.26029","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.26029"},"evidence":{"snippet":"We introduce TriViewBench, a controlled three-view visual reasoning benchmark constructed from synthetic 3D scenes with explicitly parameterized object count and occlusion.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26029"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TriViewBench evaluates multimodal LLMs on multi-view structural reasoning using synthetic 3D scenes with controlled object count and occlusion. It comprises 1,923 scenes and over 14,000 QA pairs across four complexity levels and three reasoning categories: Local Decision, Object Counting, and Global Recovery.","whyItMatters":"TriViewBench provides controlled complexity scaling to isolate structural reasoning capabilities in MLLMs, revealing distinct failure modes and bottlenecks. It enables systematic comparison of models on multi-view spatial reasoning, informing targeted improvements.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c359dc6ba00b933cad70c9512f2674719f6afa83b31e3a1712fbd3a0882a4e02"},"motivation":"Multimodal Large Language Models (MLLMs) demonstrate strong performance on standard visual question answering benchmarks, yet their scalability under controlled structural complexity remains poorly understood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.26029","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_toolbench-x_cb76a568","familyId":"bmf_da7a379f3f02","name":"ToolBench-X","oneLine":"ToolBench-X evaluates tool-using agents on multi-step tasks with executable tools and automatic scoring, in environments that include recoverable reliability hazards such as specification drift, invocation errors, execution failures, output drift, and cross-source conflicts.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.25819","pdf":"https://arxiv.org/pdf/2606.25819","project":null,"code":"https://github.com/Foreverskyou/ToolBench-X","data":null,"hfPaper":"https://huggingface.co/papers/2606.25819"},"evidence":{"snippet":"We introduce ToolBench-X, a benchmark for evaluating agents under recoverable reliability hazards.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.25819"},"ranking":{"90d":{"score":39,"rank":302,"coverage":0.7,"confidence":"Medium"}},"description":"ToolBench-X evaluates tool-using agents on multi-step tasks with executable tools and automatic scoring, in environments that include recoverable reliability hazards such as specification drift, invocation errors, execution failures, output drift, and cross-source conflicts.","whyItMatters":"Existing tool-use benchmarks assume stable tool environments, leaving a gap in evaluating agent performance under realistic unreliability. ToolBench-X provides a way to measure and compare agents' ability to diagnose and recover from tool hazards, which is crucial for practical deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"47490e3bfa5efc4a4f029089bdfb5b7c29fa6ed5d1611883b19f16895ce9da12"},"motivation":"Large language models are increasingly deployed as agents that solve tasks by interacting with external tool environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.25819","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ToolBench-X Team","organizationType":"academic-lab","sourceUrl":"https://github.com/Foreverskyou/ToolBench-X","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_tacverse_8b86d6a1","familyId":"bmf_876804079426","name":"TacVerse","oneLine":"TacVerse is a multi-sensor dataset and benchmark for cross-sensor vision-based tactile perception, containing 106,800 tactile images from seven vision-based tactile sensors. It supports shape classification, grating classification, and force regression tasks, with evaluation under within-sensor, zero-shot cross-sensor, and few-shot adaptation settings.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.RO"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.25877","pdf":"https://arxiv.org/pdf/2606.25877","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.25877"},"evidence":{"snippet":"We present TacVerse, a multi-sensor dataset and benchmark for cross-sensor vision-based tactile perception.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.25877"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TacVerse is a multi-sensor dataset and benchmark for cross-sensor vision-based tactile perception, containing 106,800 tactile images from seven vision-based tactile sensors. It supports shape classification, grating classification, and force regression tasks, with evaluation under within-sensor, zero-shot cross-sensor, and few-shot adaptation settings.","whyItMatters":"TacVerse fills a gap in evaluating generalization across tactile sensor designs, which is crucial for real-world robot deployment. It enables systematic study of sensor shift, data-efficient adaptation, and self-supervised learning in tactile perception, providing a controlled testbed for improving cross-sensor robustness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"56e813bce40f06b11b51293f939370ed7a5a66828574ff70403ea2c4ba00709e"},"motivation":"Vision-based tactile sensors (VBTSs) enable robots to infer contact geometry and force-related cues by imaging deformation through an internal camera, yet generalisation across sensor designs remains poorly understood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.25877","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ssmnbench_555dd856","familyId":"bmf_9224c6cbf216","name":"SSMNBench","oneLine":"SSMNBench is a diagnostic benchmark for cross-view human and human-object understanding, comprising 3,300 QA pairs categorized into Single-View Sufficiency (SVS) and Multi-View Necessity (MVN) tasks. It evaluates multimodal LLMs by perturbing view availability to assess distraction robustness and cross-view evidence integration.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.25634","pdf":"https://arxiv.org/pdf/2606.25634","project":null,"code":"https://github.com/gtc-gh/SSMNBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.25634"},"evidence":{"snippet":"To address this issue, we introduce SSMNBench, a diagnostic benchmark comprising 3,300 curated QA pairs for cross-view human and human-object understanding.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.25634"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SSMNBench is a diagnostic benchmark for cross-view human and human-object understanding, comprising 3,300 QA pairs categorized into Single-View Sufficiency (SVS) and Multi-View Necessity (MVN) tasks. It evaluates multimodal LLMs by perturbing view availability to assess distraction robustness and cross-view evidence integration.","whyItMatters":"Addresses the gap in evaluating genuine cross-view synthesis versus reliance on single-image semantics in MLLMs. It provides a rigorous framework for diagnosing limitations in cross-view understanding, guiding development of multimodal architectures for complex scenes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4a9df5d8069cd7572a4b03dc3a5607df0f30dcb4599eb3349614eab789a59594"},"motivation":"Multimodal Large Language Models (MLLMs) have shown remarkable progress in single-image perception, yet their ability to reason about complex cross-view human-centric scenes remains largely unverified.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.25634","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_scidraw-bench_755dfcbf","familyId":"bmf_6c84b0b5c404","name":"SciDraw-Bench","oneLine":"SciDraw-Bench is a benchmark for scientific figure generation, with 32 tasks across eight figure types and ten disciplines. Each task pairs a natural-language prompt with a machine-checkable specification, and evaluation uses four dimensions: Text Fidelity, Semantic Correctness, Structural Quality, and Convention Adherence.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.28406","pdf":"https://arxiv.org/pdf/2606.28406","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.28406"},"evidence":{"snippet":"We introduce SciDraw-Bench, a benchmark of 32 structured scientific-figure generation tasks spanning eight figure types and ten disciplines, where each task pairs a natural-language prompt with a machine-checkable specification of required labels, relations, components, conventions, and negative constraints.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.28406"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SciDraw-Bench is a benchmark for scientific figure generation, with 32 tasks across eight figure types and ten disciplines. Each task pairs a natural-language prompt with a machine-checkable specification, and evaluation uses four dimensions: Text Fidelity, Semantic Correctness, Structural Quality, and Convention Adherence.","whyItMatters":"Fills the gap in evaluating scientific figure generation, which requires correct labels and diagrammatic structure beyond natural-image composition. It provides a protocol for measuring usability of generated figures, aiding development of domain-specific generative systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dd1e6f68293f117d665d0b51d23ab571b6b6975f15148dc77a58cb2fe14f239a"},"motivation":"Text-to-image and multimodal generative models are increasingly used to produce scientific figures such as mechanism diagrams, experimental-design schematics, conceptual frameworks, and graphical abstracts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.28406","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_revengebench_23ac692d","familyId":"bmf_928072ade2b4","name":"RevengeBench","oneLine":"RevengeBench is a benchmark for recovering code-space policies from behavioral traces. It includes 75 LLM-generated policies across five game environments, where a learner designs behavioral probes and submits executable hypotheses, evaluated using continuous action-distance metrics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.26094","pdf":"https://arxiv.org/pdf/2606.26094","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.26094"},"evidence":{"snippet":"We introduce RevengeBench, a benchmark of 75 LLM generated, Elo-calibrated policies across five game environments, drawn from CodeClash tournament trajectories.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26094"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"RevengeBench is a benchmark for recovering code-space policies from behavioral traces. It includes 75 LLM-generated policies across five game environments, where a learner designs behavioral probes and submits executable hypotheses, evaluated using continuous action-distance metrics.","whyItMatters":"Addresses the inverse problem of inferring hidden decision programs from observations, relevant to opponent modeling and policy interpretability. It provides a tractable testbed for studying how controlled experiments improve code-space recovery.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"42b247ad81d5bd6e9df43d160eb22e5216533c64ab9133092e848e1d86399656"},"motivation":"For most of scientific history, researchers studying behavior could only infer hidden mechanisms from outward actions: an inverse problem that becomes more tractable when observation is augmented by targeted intervention.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.26094","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_obshazard-bench_aa08033a","familyId":"bmf_697b42360627","name":"Obshazard-bench","oneLine":"Obshazard-bench is a real-time benchmark for disaster intelligence, integrating raw satellite and ground-station data. It covers 8 disaster categories and 28 sub-categories across 60+ countries, with VQA samples and a three-stage evaluation taxonomy: Predictive Crisis Anticipation, Active Evolution Reasoning, and Multi-faceted Impact Quantification.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00012","pdf":"https://arxiv.org/pdf/2608.00012","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00012"},"evidence":{"snippet":"To bridge this gap, we introduce Obshazard-bench, a real-time, observation-driven benchmark for evaluating disaster intelligence in MLLMs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00012"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Obshazard-bench is a real-time benchmark for disaster intelligence, integrating raw satellite and ground-station data. It covers 8 disaster categories and 28 sub-categories across 60+ countries, with VQA samples and a three-stage evaluation taxonomy: Predictive Crisis Anticipation, Active Evolution Reasoning, and Multi-faceted Impact Quantification.","whyItMatters":"Fills the gap in evaluating MLLMs for operational disaster response, which require real-time reasoning from raw observation streams. It provides a realistic testbed for assessing decision-support capabilities in evolving emergencies.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"52071dd3e129915c8dee4b0baa8e5a5714fed0165eef5d197a01434a8343478e"},"motivation":"Multimodal Large Language Models (MLLMs) are increasingly used to interpret Earth observation data, yet their capability to support real-world disaster emergency response remains insufficiently evaluated.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00012","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mkg-rag-bench_ba53e1c9","familyId":"bmf_7f5057aa0709","name":"MKG-RAG-Bench","oneLine":"MKG-RAG-Bench is a benchmark for retrieval in multimodal knowledge graph-augmented generation, constructed from two knowledge graphs (general and medical) with QA datasets supporting controlled evaluation of retrieval and generation. It uses an LLM-based curation pipeline for structurally grounded queries.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval","Factuality"],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.26458","pdf":"https://arxiv.org/pdf/2606.26458","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.26458"},"evidence":{"snippet":"To address this gap, we introduce MKG-RAG-Bench, a cross-domain benchmark explicitly designed to evaluate retrieval in MKG-RAG.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26458"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MKG-RAG-Bench is a benchmark for retrieval in multimodal knowledge graph-augmented generation, constructed from two knowledge graphs (general and medical) with QA datasets supporting controlled evaluation of retrieval and generation. It uses an LLM-based curation pipeline for structurally grounded queries.","whyItMatters":"Isolates retrieval as a first-class evaluation target in MKG-RAG, addressing the challenge of heterogeneous multimodal knowledge. It provides a foundation for diagnosing retrieval limitations and improving end-to-end RAG systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4b4f1f01a0b0f9f02a4ee2efaa16e948f1c361424466ab402743574d523afae4"},"motivation":"Retrieval-augmented generation (RAG) over knowledge graphs has emerged as a promising approach for grounding large language models, yet existing benchmarks largely overlook the challenges of retrieval in multimodal knowledge graph RAG (MKG-RAG).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"KDD'26","evidence":"Accepted by KDD'26","evidenceUrl":"https://arxiv.org/abs/2606.26458","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"KDD'26","reviewStatus":"accepted","decisionRaw":"Accepted by KDD'26","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.26458","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by KDD'26","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception","Search & Retrieval"],"domainScope":"general"},{"id":"bm_libevobench_4c11745f","familyId":"bmf_0e8e47aebe59","name":"LibEvoBench","oneLine":"LibEvoBench is a multi-task benchmark for evaluating code generation models on API evolution across versions of Python libraries. It includes tasks spanning multiple library versions and introduces the Software Evolution Understanding Score (SEUS) to measure version-specific knowledge consistency.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation","Factuality"],"topics":["Code"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.25402","pdf":"https://arxiv.org/pdf/2606.25402","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.25402"},"evidence":{"snippet":"To systematically evaluate this phenomenon, we introduce LibEvoBench, a multi-task benchmark spanning multiple versions of widely used Python libraries, along with a new metric, the Software Evolution Understanding Score (SEUS), to measure models' consistency when working with evolving APIs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.25402"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LibEvoBench is a multi-task benchmark for evaluating code generation models on API evolution across versions of Python libraries. It includes tasks spanning multiple library versions and introduces the Software Evolution Understanding Score (SEUS) to measure version-specific knowledge consistency.","whyItMatters":"Addresses the gap in evaluating models on version-specific API knowledge, which is critical for real-world software maintenance. It highlights limitations of current training on temporally mixed corpora and motivates temporally grounded learning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7ae1075acff0c0c02977d3297cabbb264cd8d1fa8a08ba414458daa3221d339d"},"motivation":"Large software projects often depend on older versions of libraries, even as APIs continue to evolve across releases.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"DL4Code workshop at ICML 2026","evidence":"Accepted at the DL4Code workshop at ICML 2026","evidenceUrl":"https://arxiv.org/abs/2606.25402","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"DL4Code workshop at ICML 2026","reviewStatus":"accepted","decisionRaw":"Accepted at the DL4Code workshop at ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.25402","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at the DL4Code workshop at ICML 2026","level":"author-claim"}]}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_investphilbench_547d0c00","familyId":"bmf_2ef9232fbd60","name":"InvestPhilBench","oneLine":"InvestPhilBench is a multi-layer benchmark for evaluating procedural reasoning in investment philosophy, spanning eight cognitive tiers from principle identification to novel framework extrapolation. It includes principle cards, decision-framework cards, and QA questions, with an automated scoring pipeline (BASP) and five algorithmic metrics.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.25984","pdf":"https://arxiv.org/pdf/2606.25984","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.25984"},"evidence":{"snippet":"We introduce InvestPhilBench, a multi-layer benchmark spanning eight cognitive tiers, from principle identification (L1) to novel framework extrapolation (L8).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.25984"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"InvestPhilBench is a multi-layer benchmark for evaluating procedural reasoning in investment philosophy, spanning eight cognitive tiers from principle identification to novel framework extrapolation. It includes principle cards, decision-framework cards, and QA questions, with an automated scoring pipeline (BASP) and five algorithmic metrics.","whyItMatters":"Fills the gap in testing whether LLMs can accurately reconstruct and apply expert procedural decision frameworks. Provides a reproducible method for scoring procedural reasoning, with a metric (GRA) that exposes deficits hidden by composite scores.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"40867a345cfe01336a8ae3495be03bad7db765cf78bb971da89c034bca5b94ef"},"motivation":"Large language models are increasingly deployed as investment research assistants, yet no benchmark tests whether they can accurately reconstruct and apply the specific procedural decision frameworks of expert investors.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.25984","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_hg-bench_1e72857e","familyId":"bmf_5e2cc69b8f13","name":"HG-Bench","oneLine":"HG-Bench evaluates page-aware, two-level answer-region grounding: given multi-page handwritten homework images, models must localize complete answer regions and ordered step-level subregions, with question- and step-level boxes under a hierarchical constraint.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.25491","pdf":"https://arxiv.org/pdf/2606.25491","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.25491"},"evidence":{"snippet":"We introduce HG-Bench, a benchmark of 500 human-annotated K-12 homework samples curated from a 1,489,278-image source pool, with question-level and step-level boxes linked by a hierarchical containment constraint.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.25491"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HG-Bench evaluates page-aware, two-level answer-region grounding: given multi-page handwritten homework images, models must localize complete answer regions and ordered step-level subregions, with question- and step-level boxes under a hierarchical constraint.","whyItMatters":"Automated homework assessment needs both answer recognition and spatial grounding of reasoning steps. Prior benchmarks miss page-aware, multi-level localization; HG-Bench provides a reproducible protocol to measure this capability gap across models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"39c9fa1786310ab51bdd5cc4043c56cafc1d7cb28dc7a2124c791f16256ecc89"},"motivation":"Automated homework assessment depends not only on recognizing student answers, but also on accurately locating where each answer and each intermediate reasoning step appears in noisy, multi-page handwritten work.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.25491","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_funpiq_ac17b6fd","familyId":"bmf_d55ffcdf5c6f","name":"FunPiQ","oneLine":"FunPiQ is a benchmark for pixel-level fundus image quality assessment with pixel-level annotations for anatomical visibility, using a three-class scheme (good, usable, bad). It also introduces EFIQA-CP, an explainable method trained with pseudo-labels.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.25915","pdf":"https://arxiv.org/pdf/2606.25915","project":null,"code":"https://github.com/penway/FunPiQ","data":null,"hfPaper":"https://huggingface.co/papers/2606.25915"},"evidence":{"snippet":"In this work, we introduce FunPiQ, the first FIQA benchmark to provide pixel-level quality annotations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.25915"},"ranking":{"90d":{"score":39,"rank":270,"coverage":0.7,"confidence":"Medium"}},"description":"FunPiQ is a benchmark for pixel-level fundus image quality assessment with pixel-level annotations for anatomical visibility, using a three-class scheme (good, usable, bad). It also introduces EFIQA-CP, an explainable method trained with pseudo-labels.","whyItMatters":"Existing FIQA benchmarks provide only image-level labels, limiting localized degradation analysis. FunPiQ enables task-agnostic explainable quality evaluation through pixel-level ground truth and offers a public dataset for method comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cb7daff2c922718533100460e5fa5b1c1c9fe3e8e47e882df05bab32cf186c94"},"motivation":"Color fundus photography (CFP) is the most common ophthalmic imaging modality for large-scale screening.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"MICCAI 2026 main conference","evidence":"Accepted at MICCAI 2026 main conference. Our code, weights, and dataset are available at https://github.com/penway/FunPiQ","evidenceUrl":"https://arxiv.org/abs/2606.25915","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"MICCAI 2026 main conference","reviewStatus":"accepted","decisionRaw":"Accepted at MICCAI 2026 main conference. Our code, weights, and dataset are available at https://github.com/penway/FunPiQ","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.25915","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at MICCAI 2026 main conference. Our code, weights, and dataset are available at https://github.com/penway/FunPiQ","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_finbench_34ca17f0","familyId":"bmf_5ee7524fb0ac","name":"FinBench","oneLine":"FinBench is a benchmark for evaluating calibration and uncertainty in financial forecasting with time-gated tasks, requiring probability of positive return and 80% prediction interval, scored with Brier and Winkler scores.","area":"Safety & Trustworthiness","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["stat.AP"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.16229","pdf":"https://arxiv.org/pdf/2607.16229","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.16229"},"evidence":{"snippet":"We introduce FinBench, a benchmark designed to evaluate calibration and uncertainty quality for financial forecasting in a setting that is (i) strictly time-gated to avoid look-ahead bias and (ii) evaluated with strictly proper scoring rules that penalize hallucinated confidence.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.16229"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FinBench is a benchmark for evaluating calibration and uncertainty in financial forecasting with time-gated tasks, requiring probability of positive return and 80% prediction interval, scored with Brier and Winkler scores.","whyItMatters":"Financial forecasting agents risk overconfidence; FinBench addresses this by testing probabilistic calibration under temporal constraints, but its pilot scale limits immediate comparison value.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2f20475c33eaeac878e03ec82af798089271947533fa51c38bab9656d02aeecf"},"motivation":"Large language models (LLMs) are increasingly used as components of agentic systems that observe, plan, and act.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.16229","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_cyberchainbench_89e4f090","familyId":"bmf_073d2b7764c1","name":"CyberChainBench","oneLine":"CyberChainBench evaluates LLM-based agents on smart contract security across vulnerability detection, exploit generation, and patch synthesis, using 541 real exploit incidents with on-chain evaluation on historical forks.","area":"Vision & 3D","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26216","pdf":"https://arxiv.org/pdf/2606.26216","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.26216"},"evidence":{"snippet":"We present CyberChainBench, a benchmark for evaluating LLM-based agents on smart contract security across three complementary tasks: vulnerability detection, exploit generation, and patch synthesis.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26216"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CyberChainBench evaluates LLM-based agents on smart contract security across vulnerability detection, exploit generation, and patch synthesis, using 541 real exploit incidents with on-chain evaluation on historical forks.","whyItMatters":"Smart contract security requires realistic, end-to-end evaluation of agent capabilities; CyberChainBench provides structured ground truth and economic impact metrics to measure practical effectiveness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"372a1ccbf58b9582df8654725de82f7b7ee7945ae56f4e91c642fe524ce71754"},"motivation":"We present CyberChainBench, a benchmark for evaluating LLM-based agents on smart contract security across three complementary tasks: vulnerability detection, exploit generation, and patch synthesis.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.26216","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_conflictbench_d8fc3760","familyId":"bmf_73bdd16dbae8","name":"ConflictBench","oneLine":"ConflictBench is a benchmark with ConflictScore metric to quantify how well models acknowledge conflicting evidence in grounding documents, decomposing responses into claims and labeling them against documents.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.26437","pdf":"https://arxiv.org/pdf/2606.26437","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.26437"},"evidence":{"snippet":"We develop ConflictBench, a benchmark covering diverse forms of conflicts such as ambiguity, contradiction, and divergent opinions, to systematically evaluate our metric.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.26437"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"ConflictBench is a benchmark with ConflictScore metric to quantify how well models acknowledge conflicting evidence in grounding documents, decomposing responses into claims and labeling them against documents.","whyItMatters":"Existing factuality metrics ignore coexisting contradictions; ConflictScore provides a nuanced measure over ConflictBench, offering a corrective feedback mechanism for improving truthfulness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"699c153bdab56cf6cd71b2fe4143d3585cbbeaaca8334edfa50d2e2e55c5aa86"},"motivation":"Existing metrics for factuality and faithfulness evaluate whether an answer is supported or contradicted by its grounding documents, but they fail to capture when both supporting and contradicting evidence coexist.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.26437","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_captureguide-bench_fb3878c6","familyId":"bmf_d80d29d0b93a","name":"CaptureGuide-Bench","oneLine":"CaptureGuide-Bench evaluates capture-time photography guidance with two tasks: photographer-side composition decision/refinement and subject-side scene-conditioned pose recommendation, using metrics like IoU and plausibility.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.25763","pdf":"https://arxiv.org/pdf/2606.25763","project":"https://lijayutnt.github.io/ShutterMuse","code":"https://github.com/lijayuTnT/ShutterMuse","data":null,"hfPaper":"https://huggingface.co/papers/2606.25763"},"evidence":{"snippet":"To address this gap, we introduce CaptureGuide-Bench, a benchmark with two complementary tasks: photographer-side composition decision and refinement, and subject-side scene-conditioned pose recommendation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":46,"hfDailySubmittedAt":null,"githubStars":98,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.25763"},"ranking":{"90d":{"score":80,"rank":22,"coverage":0.7,"confidence":"Medium"}},"description":"CaptureGuide-Bench evaluates capture-time photography guidance with two tasks: photographer-side composition decision/refinement and subject-side scene-conditioned pose recommendation, using metrics like IoU and plausibility.","whyItMatters":"Existing aesthetic benchmarks focus on post-hoc cropping; CaptureGuide-Bench addresses missing capture-time guidance for both framing and subject pose, providing a structured dataset for development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4460e590ef6bd3ff14cfd2afd21bb19ed0562be30bb15c8446d1711ff37a277d"},"motivation":"Real-world photography requires capture-time guidance for both camera framing and subject pose.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.25763","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_c3-bench_c82d4fcd","familyId":"bmf_8434d033ab49","name":"C3-Bench","oneLine":"C3-Bench evaluates context-aware change captioning with 4,996 human-labeled image pairs across 51 real-world contexts, using an LLM-as-Judge framework scoring correctness, specificity, fluency, relevance, and a reversibility metric.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.25445","pdf":"https://arxiv.org/pdf/2606.25445","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.25445"},"evidence":{"snippet":"To fill this gap, we propose C3-Bench, a comprehensive benchmark for evaluating Context-aware Change Captioning.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.25445"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"C3-Bench evaluates context-aware change captioning with 4,996 human-labeled image pairs across 51 real-world contexts, using an LLM-as-Judge framework scoring correctness, specificity, fluency, relevance, and a reversibility metric.","whyItMatters":"Change captioning performance varies with domain; C3-Bench exposes systematic failures in conventional models and LMMs, providing a comprehensive benchmark to drive generalization and trustworthiness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"30ec2ca6804caee6ea6be21a47bec4b8e687d6de1e05519e737c92956db04115"},"motivation":"While Change Captioning systems have garnered substantial attention to respond to our evolving world, their true performance on diverse real-world change contexts remains largely unexplored due to the lack of comprehensive evaluation frameworks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.25445","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_tailor-bench_613b036f","familyId":"bmf_a1f4ed1db648","name":"Tailor-Bench","oneLine":"Tailor-Bench evaluates visual world models on simulating irregular physical interactions with three scenario modes (regular, unconventional, impossible) and predictive/descriptive generation settings.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24256","pdf":"https://arxiv.org/pdf/2606.24256","project":null,"code":"https://github.com/tailor-bench/code","data":null,"hfPaper":"https://huggingface.co/papers/2606.24256"},"evidence":{"snippet":"In this work, we introduce Tailor-Bench, a benchmark that challenges world models to simulate irregular physical interactions.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":43,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24256"},"ranking":{"90d":{"score":35,"rank":305,"coverage":0.7,"confidence":"Medium"}},"description":"Tailor-Bench evaluates visual world models on simulating irregular physical interactions with three scenario modes (regular, unconventional, impossible) and predictive/descriptive generation settings.","whyItMatters":"Current benchmarks focus on common interactions; Tailor-Bench exposes the long-tail gap in physical world modeling, testing generalization and constraint awareness beyond typical scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c52a5c3176cb1aaeda5789a3e911165d98cebdc86bb78e307c888f7694d5ceb4"},"motivation":"Physical interactions follow a long-tailed distribution: a set of common and regular interactions dominates human experience and visual data, while a broad spectrum of rare and irregular interactions remains underrepresented.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24256","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_t2d-bench_6b92ee7c","familyId":"bmf_ece4accf1aab","name":"T2D-Bench","oneLine":"T2D-Bench evaluates LLM outputs for type 2 diabetes against evidence constraints using a multi-layer clinical-lifestyle knowledge graph, covering diagnosis, medication safety, and lifestyle conflicts across 100 structured vignettes.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Factuality"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24145","pdf":"https://arxiv.org/pdf/2606.24145","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.24145"},"evidence":{"snippet":"We present T2D-Bench, a reproducible benchmark and evidence-gated evaluation framework for testing whether LLM outputs satisfy explicit, graph-checkable evidence requirements.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24145"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"T2D-Bench evaluates LLM outputs for type 2 diabetes against evidence constraints using a multi-layer clinical-lifestyle knowledge graph, covering diagnosis, medication safety, and lifestyle conflicts across 100 structured vignettes.","whyItMatters":"Addresses the gap in evaluating whether LLM recommendations satisfy explicit clinical guidelines and justify lifestyle-related glycemic claims, providing a mechanism to detect unsupported omissions and improve verifier-level compliance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"15c2d1bbec2fdc0844a712ad7ee5c89bf538628056cfea5a1ff4d76fdc364dbd"},"motivation":"Large language models (LLMs) can produce clinically fluent recommendations for type 2 diabetes while failing to satisfy guideline constraints or explicitly justify lifestyle-related glycemic claims.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"as a poster at AMIA 2026 Annual Symposium","evidence":"7 pages, 2 figures, 2 tables. Accepted as a poster at AMIA 2026 Annual Symposium","evidenceUrl":"https://arxiv.org/abs/2606.24145","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"as a poster at AMIA 2026 Annual Symposium","reviewStatus":"accepted","decisionRaw":"7 pages, 2 figures, 2 tables. Accepted as a poster at AMIA 2026 Annual Symposium","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.24145","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"7 pages, 2 figures, 2 tables. Accepted as a poster at AMIA 2026 Annual Symposium","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_sp-bench_0f201b2f","familyId":"bmf_bba6eaed485b","name":"SP-Bench","oneLine":"SP-Mind is an autonomous AI agent for spatial proteomics analysis, and SP-Bench is introduced for evaluation with 102 tasks across 18 categories.","area":"Agents & Tool Use","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Agents","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24235","pdf":"https://arxiv.org/pdf/2606.24235","project":null,"code":"https://github.com/tomtommyyuan/spmind","data":null,"hfPaper":"https://huggingface.co/papers/2606.24235"},"evidence":{"snippet":"To rigorously evaluate its capabilities, we introduce SP-Bench, a comprehensive benchmark spanning diverse tissue types, comprising 102 tasks across 18 distinct categories.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":277,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24235"},"ranking":{"90d":{"score":72,"rank":63,"coverage":0.7,"confidence":"Medium"}},"description":"SP-Mind is an autonomous AI agent for spatial proteomics analysis, and SP-Bench is introduced for evaluation with 102 tasks across 18 categories.","whyItMatters":"The benchmark appears tied to the agent's capabilities and lacks a standalone comparison path for other teams.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b88f8a0cb8a58596d67c4a325a6559ac049f720c984c14b1f3d1996fdae72aa3"},"motivation":"Spatial proteomics enables single-cell-resolution characterization of protein expression within tissue architecture, playing a critical role in understanding tumor microenvironments and guiding precision medicine.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026","evidence":"24 pages, 6 figures. Accepted to ICML 2026. Equal contribution by Yucheng Yuan and Yuanfeng Ji","evidenceUrl":"https://arxiv.org/abs/2606.24235","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026","reviewStatus":"accepted","decisionRaw":"24 pages, 6 figures. Accepted to ICML 2026. Equal contribution by Yucheng Yuan and Yuanfeng Ji","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.24235","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"24 pages, 6 figures. Accepted to ICML 2026. Equal contribution by Yucheng Yuan and Yuanfeng Ji","level":"author-claim"}]}],"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_parapairaudiobench_e7d0c4ed","familyId":"bmf_7892b914cd26","name":"ParaPairAudioBench","oneLine":"ParaPairAudioBench evaluates LALMs as judges for paralinguistic speech across five dimensions with 5,175 audio pairs.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SD"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.24648","pdf":"https://arxiv.org/pdf/2606.24648","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.24648"},"evidence":{"snippet":"We introduce ParaPairAudioBench, a pairwise benchmark of 5,175 audio pairs across five paralinguistic dimensions: Style, Rate, Emphasis, Age, and Gender.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24648"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ParaPairAudioBench evaluates LALMs as judges for paralinguistic speech across five dimensions with 5,175 audio pairs.","whyItMatters":"Targets fine-grained paralinguistic distinctions that prior benchmarks overlook, enabling calibration-aware assessment of judge reliability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a5089523d7a6a8813f7a7aeb60994920afc28666734033e8eabb5f1f552f6607"},"motivation":"Large Audio-Language Models (LALMs) have been widely used as judge models for the automatic evaluation of generated speech.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"Interspeech 2026","evidence":"Accepted to Interspeech 2026","evidenceUrl":"https://arxiv.org/abs/2606.24648","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"Interspeech 2026","reviewStatus":"accepted","decisionRaw":"Accepted to Interspeech 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.24648","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to Interspeech 2026","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_naturebench_ffd14b37","familyId":"bmf_d35b766e0cf9","name":"NatureBench","oneLine":"NatureBench evaluates AI coding agents on 90 tasks distilled from Nature-family publications across 6 scientific domains, scoring against each paper's reported state of the art.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24530","pdf":"https://arxiv.org/pdf/2606.24530","project":null,"code":"https://github.com/FrontisAI/NatureBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.24530"},"evidence":{"snippet":"We introduce NatureBench, a cross-discipline benchmark of 90 tasks distilled from peer-reviewed Nature-family publications, designed to evaluate whether AI coding agents can move beyond reproduction toward discovery on real scientific problems.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":66,"hfDailySubmittedAt":null,"githubStars":101,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24530"},"ranking":{"90d":{"score":81,"rank":19,"coverage":0.7,"confidence":"Medium"}},"description":"NatureBench evaluates AI coding agents on 90 tasks distilled from Nature-family publications across 6 scientific domains, scoring against each paper's reported state of the art.","whyItMatters":"Provides a standardized environment and public leaderboard to measure whether coding agents can achieve discovery-level performance on real scientific problems, addressing environment-fragmentation issues in prior benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"17b5881db23bbb713bc8a2955bc962a45b051e371c9b4ea34540417320ecdf94"},"motivation":"We introduce NatureBench, a cross-discipline benchmark of 90 tasks distilled from peer-reviewed Nature-family publications, designed to evaluate whether AI coding agents can move beyond reproduction toward discovery on real scientific problems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24530","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FrontisAI","organizationType":"company-research-lab","sourceUrl":"https://github.com/FrontisAI/NatureBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_mmed-bench-ir_04b38f2d","familyId":"bmf_00f594160510","name":"MMed-Bench-IR","oneLine":"MMed-Bench-IR evaluates multilingual medical information retrieval across 6 languages with three tasks: cross-lingual QA retrieval, concept discrimination, and evidence retrieval for RAG.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Information retrieval"],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24200","pdf":"https://arxiv.org/pdf/2606.24200","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.24200"},"evidence":{"snippet":"We introduce MMed-Bench-IR, a benchmark designed to disentangle these axes across 6 languages and three structurally heterogeneous tasks: (1) cross-lingual medical QA retrieval with 6,127 queries grounded in the Unified Medical Language System (UMLS), (2) concept discrimination over 4,975 confusion sets at three difficulty tiers, and (3) multilingual evidence retrieval for RAG with 2,040 quality-assured queries.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24200"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MMed-Bench-IR evaluates multilingual medical information retrieval across 6 languages with three tasks: cross-lingual QA retrieval, concept discrimination, and evidence retrieval for RAG.","whyItMatters":"Uncovers severe cross-lingual failures in biomedical encoders that English-only benchmarks miss, highlighting the need for multilingual capability measurement in clinical RAG.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1eccf7d331acf480dc1cb1010e929f438b076b6eb13cac6834138aa764c5ad35"},"motivation":"Retrieval-augmented generation (RAG) in clinical settings increasingly requires multilingual retrieval against predominantly English evidence corpora.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24200","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"specific"},{"id":"bm_memprobe_9e043327","familyId":"bmf_8b0112f7dc9d","name":"MEMPROBE","oneLine":"MEMPROBE evaluates long-term memory in LLM agents by reconstructing hidden user-state from agent memory across 50 simulated users with 31 hidden dimensions each.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.24595","pdf":"https://arxiv.org/pdf/2606.24595","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.24595"},"evidence":{"snippet":"We instantiate this view in MEMPROBE, a benchmark in which a memory-equipped agent assists simulated users, each carrying a hidden, taxonomy-anchored user-state bank, across a trajectory of leak-controlled tasks, after which that bank is reconstructed from the agent's resulting memory under both full-store and top-k access.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24595"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MEMPROBE evaluates long-term memory in LLM agents by reconstructing hidden user-state from agent memory across 50 simulated users with 31 hidden dimensions each.","whyItMatters":"Offers a direct measure of memory fidelity as an auditable artifact, distinct from downstream task success, potentially improving agent memory evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e3b0d2b5d0a4c211b19e81ca11157d54d040ed43f738876f5848b936945f3209"},"motivation":"Long-term memory promises LLM agents that grow more capable across sessions, maintaining an accurate, evolving understanding of the user that interaction forms.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24595","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_medbench_c5b43204","familyId":"bmf_9629d422c4d1","name":"MedBench","oneLine":"MedBench v5 evaluates clinical multimodal models across 63 tasks with process-oriented metrics including stressors and hallucination tracking.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Agents","Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24155","pdf":"https://arxiv.org/pdf/2606.24155","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.24155"},"evidence":{"snippet":"We introduce MedBench v5, a redesigned benchmark for clinical multimodal models (language, vision-language, and agent systems) that moves from static QA to dynamic, process-oriented evaluation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24155"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MedBench v5 evaluates clinical multimodal models across 63 tasks with process-oriented metrics including stressors and hallucination tracking.","whyItMatters":"Addresses gaps in process visibility and hallucination detection in medical AI evaluation, offering a unified framework for capability profiling and stress testing.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3f5ec9ae3c543561fed00836ed0e0126898a6bbee8aa121270d4d71de598a871"},"motivation":"Existing medical AI benchmarks lack process visibility, atomic skill evaluation, and integrated hallucination detection.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24155","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_geot2v-bench_948c1065","familyId":"bmf_c3ac22233fcc","name":"GeoT2V-Bench","oneLine":"GeoT2V-Bench evaluates 3D consistency in camera-prompted text-to-video models via 3D reconstruction, using metrics like static rendering error and flow agreement.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.24829","pdf":"https://arxiv.org/pdf/2606.24829","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.24829"},"evidence":{"snippet":"We introduce GeoT2V-Bench, a reconstruction-based diagnostic benchmark for evaluating whether camera-prompted T2V clips can support explicit rigid 3D reconstruction.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24829"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GeoT2V-Bench evaluates 3D consistency in camera-prompted text-to-video models via 3D reconstruction, using metrics like static rendering error and flow agreement.","whyItMatters":"Assesses whether generated videos can support explicit rigid 3D reconstruction, providing a diagnostic tool for a critical limitation of T2V models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"15f09e26d6e0ac8fd9d968007cc0ce0064cd932296321c4884429740de727f04"},"motivation":"Camera-prompted text-to-video (T2V) models are increasingly used to synthesize virtual camera captures, such as orbiting objects or moving through static scenes.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24829","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_evoclawbench_03620568","familyId":"bmf_3bf0a17e8b5d","name":"EvoClawBench","oneLine":"EvoClawBench evaluates whether an agent runtime can convert evidence from its own runs into reusable skills that improve fresh executions. It covers 100 tasks and 502 sub-problems across coding, data, office, security, operations, and domain-document workflows, comparing direct execution, pre-authored skills, and post-run skill summarization.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.09711","pdf":"https://arxiv.org/pdf/2607.09711","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.09711"},"evidence":{"snippet":"We introduce EvoClawBench, a benchmark for this closed-loop skill-learning question on repeated, fixture-backed tasks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.09711"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"EvoClawBench evaluates whether an agent runtime can convert evidence from its own runs into reusable skills that improve fresh executions. It covers 100 tasks and 502 sub-problems across coding, data, office, security, operations, and domain-document workflows, comparing direct execution, pre-authored skills, and post-run skill summarization.","whyItMatters":"The evaluation gap is assessing closed-loop skill learning in agents, where benefits are selective and cost-sensitive rather than automatic. The benchmark provides a decision value for runtime developers and users considering skill-authoring loops.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"245f3c84838aebaf1465c81f18340a4346f4ad460b6505d9bda65fc4b1e648ec"},"motivation":"Existing agent benchmarks primarily test task completion, tool use, or skill utility, but do not isolate whether a runtime can convert evidence from its own runs into reusable skills that improve fresh executions after authoring overhead.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.09711","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_egosat_1b88fd52","familyId":"bmf_710c21ef2296","name":"EgoSAT","oneLine":"EgoSAT evaluates vision-language models on egocentric video reasoning in streaming settings. It contains 1,997 videos (165 hours) and about 4,800 QA pairs covering retrospective, online, and prospective reasoning tasks.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24422","pdf":"https://arxiv.org/pdf/2606.24422","project":"https://leiyj23.github.io/EgoSAT/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.24422"},"evidence":{"snippet":"We introduce EgoSAT, the first comprehensive benchmark for egocentric video reasoning in streaming settings, designed to evaluate the capabilities of modern vision-language models (VLMs).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24422"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"EgoSAT evaluates vision-language models on egocentric video reasoning in streaming settings. It contains 1,997 videos (165 hours) and about 4,800 QA pairs covering retrospective, online, and prospective reasoning tasks.","whyItMatters":"Serves as a unified benchmark for streaming egocentric interaction understanding, enabling assessment of temporal reasoning and confidence calibration in VLMs, which is crucial for reliable deployment in real-time applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6a329f1bd677c01702fb8c5b6626912e87069c971f8da4966f3953260f71da1c"},"motivation":"We introduce EgoSAT, the first comprehensive benchmark for egocentric video reasoning in streaming settings, designed to evaluate the capabilities of modern vision-language models (VLMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ECCV 2026","evidence":"Accepted to ECCV 2026. Project page: https://leiyj23.github.io/EgoSAT/","evidenceUrl":"https://arxiv.org/abs/2606.24422","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ECCV 2026","reviewStatus":"accepted","decisionRaw":"Accepted to ECCV 2026. Project page: https://leiyj23.github.io/EgoSAT/","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.24422","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to ECCV 2026. Project page: https://leiyj23.github.io/EgoSAT/","level":"author-claim"}]}],"publishers":[{"name":"EgoSAT Project","organizationType":"academic-lab","sourceUrl":"https://leiyj23.github.io/EgoSAT/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_cn-newstts-bench_e74cef36","familyId":"bmf_c1b15a4e7ff6","name":"CN-NewsTTS Bench","oneLine":"CN-NewsTTS Bench v0.1 evaluates Chinese news TTS pronunciation of dense written forms (scores, model names, ranges, units, percentages, abbreviations) from raw text, using 800 public test records and a 992-target auto-evaluable subset with ASR-ensemble transcripts and an automatic target scorer.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24714","pdf":"https://arxiv.org/pdf/2606.24714","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.24714"},"evidence":{"snippet":"We introduce CN-NewsTTS Bench v0.1, an open target-level benchmark for evaluating whether Chinese news TTS products pronounce such targets correctly from raw text, without user-side rules, LLM rewriting, SSML hints, or manual edits.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24714"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CN-NewsTTS Bench v0.1 evaluates Chinese news TTS pronunciation of dense written forms (scores, model names, ranges, units, percentages, abbreviations) from raw text, using 800 public test records and a 992-target auto-evaluable subset with ASR-ensemble transcripts and an automatic target scorer.","whyItMatters":"Addresses the gap where TTS systems may preserve written strings while altering spoken meaning in Chinese news. Provides a reproducible target-level scoring contract for comparing pronunciation accuracy across systems without manual intervention.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"38c0273a14823ec91b9d167248829d6271e3703be38dc2f02ba287f2596a65e2"},"motivation":"Chinese news text contains dense written forms such as scores, hyphenated model names, ranges, unit symbols, percentages, English abbreviations, and mixed Chinese-Latin-digit names.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24714","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_behaviorbench_01c04f85","familyId":"bmf_2d286f1be0d2","name":"BehaviorBench","oneLine":"Evaluates foundation models on four behavioral science capabilities: behavior prediction and simulation, strategic decision-making, subject-trait inference, and behavioral knowledge application, assessing both individual-level and distributional alignment.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24162","pdf":"https://arxiv.org/pdf/2606.24162","project":"https://umich-foreseer.github.io/behaviorbench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.24162"},"evidence":{"snippet":"We introduce BehaviorBench, a comprehensive benchmark that evaluates foundation models along four core capabilities: (1) behavior prediction and simulation, (2) strategic decision-making, (3) subject-trait inference, and (4) behavioral knowledge application.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24162"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates foundation models on four behavioral science capabilities: behavior prediction and simulation, strategic decision-making, subject-trait inference, and behavioral knowledge application, assessing both individual-level and distributional alignment.","whyItMatters":"Addresses the lack of systematic evaluation of foundation models in behavioral science, providing a standardized benchmark that captures population-level validity, helping researchers and practitioners choose models for behavioral simulation and analysis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d6eff0c3c37a7bb9add1c0946432270af6262242cf25f20be88b1394976141e9"},"motivation":"Foundation models have been increasingly applied to behavioral science domains such as psychology, sociology, and economics.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24162","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"University of Michigan Foreseer Lab","organizationType":"academic-lab","sourceUrl":"https://umich-foreseer.github.io/behaviorbench/","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_agora_fee07ca2","familyId":"bmf_f7070d57bbe5","name":"AGORA","oneLine":"Agora evaluates agentic document reasoning across eight domain collections of 9,664 authentic workplace documents. It includes 362 questions requiring location of sparse evidence and reconciliation of terminology, units, and time conventions. The benchmark is designed to exceed model context windows, necessitating deliberate exploration.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24526","pdf":"https://arxiv.org/pdf/2606.24526","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.24526"},"evidence":{"snippet":"We introduce Agora, a benchmark pairing 362 questions with eight domain collections of 9,664 authentic documents and 372M tokens, far exceeding any model's context window, so agents must explore deliberately rather than scan exhaustively.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":5,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24526"},"ranking":{"90d":{"score":52,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Agora evaluates agentic document reasoning across eight domain collections of 9,664 authentic workplace documents. It includes 362 questions requiring location of sparse evidence and reconciliation of terminology, units, and time conventions. The benchmark is designed to exceed model context windows, necessitating deliberate exploration.","whyItMatters":"Agora addresses the gap in evaluating archive-grounded reasoning where agents must navigate large, messy document collections. It provides a challenging and realistic testbed for assessing agentic document search and synthesis capabilities, offering practical insights for deployment in document-intensive domains.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"78100f1519a9c322586b53a6b603f591e87a8da3920f7d645914a49eb2abf118"},"motivation":"Large language models are increasingly deployed as agents that reason over documents rather than answer from parametric knowledge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24526","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AGORA Benchmark Team","organizationType":"benchmark-organization","sourceUrl":"https://arxiv.org/abs/2606.24526","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_agentworldbench_775e3273","familyId":"bmf_2508b6fd7ca7","name":"AgentWorldBench","oneLine":"AgentWorldBench evaluates language world models on simulation fidelity across 7 domains (MCP, Search, Terminal, SWE, Android, Web, OS) using real-world trajectories and rubric-based scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.24597","pdf":"https://arxiv.org/pdf/2606.24597","project":null,"code":"https://github.com/QwenLM/Qwen-AgentWorld","data":null,"hfPaper":"https://huggingface.co/papers/2606.24597"},"evidence":{"snippet":"To evaluate language world models, we present AgentWorldBench, a comprehensive benchmark constructed from real-world interactions of 5 frontier models on 9 established benchmarks.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":159,"hfDailySubmittedAt":null,"githubStars":965,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24597"},"ranking":{"90d":{"score":85,"rank":6,"coverage":0.7,"confidence":"Medium"}},"description":"AgentWorldBench evaluates language world models on simulation fidelity across 7 domains (MCP, Search, Terminal, SWE, Android, Web, OS) using real-world trajectories and rubric-based scoring.","whyItMatters":"This benchmark addresses the lack of systematic evaluation for language world models in agentic environments, providing a standardized protocol that enables direct comparison and guides development of general agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d2f0149d859e2cc3718f357e72bdc1a4b58e0d7c89f4f4e6cdffa38eb0cc7176"},"motivation":"A world model predicts environment dynamics based on current observations and actions, serving as a core cognitive mechanism for reasoning and planning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24597","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_agenticinterpbench_4e5baf78","familyId":"bmf_6546f2911c91","name":"AgenticInterpBench","oneLine":"Evaluates language model agents on explaining components of transformer circuits, with 84 semi-synthetic circuits and 163 component-level annotations.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24026","pdf":"https://arxiv.org/pdf/2606.24026","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.24026"},"evidence":{"snippet":"We introduce AgenticInterpBench, a benchmark for circuit explanation built from 84 semi-synthetic transformer circuits with 163 component-level annotations.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24026"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Evaluates language model agents on explaining components of transformer circuits, with 84 semi-synthetic circuits and 163 component-level annotations.","whyItMatters":"Addresses the lack of standardized evaluation for circuit explanation in mechanistic interpretability, but lacks a standalone public comparison path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bef0db407df995cb074583af471d5bf49e108f89564c797979c2fa8eee814c2f"},"motivation":"Mechanistic interpretability has made substantial progress in automatically localizing circuits, but explaining what localized components do remains labor-intensive and difficult to standardize.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24026","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_triggerbench_0852166e","familyId":"bmf_85eecdfdb4f5","name":"TriggerBench","oneLine":"TriggerBench is a benchmark for evaluating prospective memory (PM) in LLMs, spanning five dimensions across daily assistant and professional workflow scenarios, with matched retrospective memory (RM) controls, contrastive variants, and overloaded triggers, measuring proactive recall, false-alarm rate, and attentional robustness.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.23459","pdf":"https://arxiv.org/pdf/2606.23459","project":null,"code":"https://github.com/KristenZHANG/TriggerBench-Official","data":null,"hfPaper":"https://huggingface.co/papers/2606.23459"},"evidence":{"snippet":"We introduce TriggerBench, a comprehensive PM benchmark spanning five dimensions across both daily assistants and professional workflows.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.23459"},"ranking":{"90d":{"score":25,"rank":390,"coverage":0.7,"confidence":"Medium"}},"description":"TriggerBench is a benchmark for evaluating prospective memory (PM) in LLMs, spanning five dimensions across daily assistant and professional workflow scenarios, with matched retrospective memory (RM) controls, contrastive variants, and overloaded triggers, measuring proactive recall, false-alarm rate, and attentional robustness.","whyItMatters":"Existing LLM evaluations focus on retrospective memory via explicit queries, leaving prospective memory – the ability to spontaneously act on latent constraints – unevaluated. TriggerBench provides a granular measurement of PM capabilities, revealing a precision-recall trade-off, attentional fragility, and a decay with context length that RM does not exhibit, informing deployment decisions for long interactive applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"60a37abf835e8cac58ae5bad8246d90bf5bc54172cfcc4586079dbbfe3f46c38"},"motivation":"While Large Language Models (LLMs) are increasingly deployed in long interactions, existing evaluations focus predominantly on retrospective memory (RM) via explicit queries.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.23459","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"KristenZHANG (GitHub)","organizationType":"community","sourceUrl":"https://github.com/KristenZHANG/TriggerBench-Official","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_tf-refusalbench_fb205626","familyId":"bmf_c969544e9b3f","name":"TF-RefusalBench","oneLine":"TF-RefusalBench is a multilingual benchmark for criminal-law translation and summarization derived from public Swiss Supreme Court rulings. It contains 5,200 prompts across French, German, Italian, and English, designed to trigger refusals in LLMs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.23375","pdf":"https://arxiv.org/pdf/2606.23375","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.23375"},"evidence":{"snippet":"To measure this phenomenon, we introduce TF-RefusalBench, a multilingual benchmark for criminal-law translation and summarization derived from public Swiss Supreme Court rulings.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.23375"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TF-RefusalBench is a multilingual benchmark for criminal-law translation and summarization derived from public Swiss Supreme Court rulings. It contains 5,200 prompts across French, German, Italian, and English, designed to trigger refusals in LLMs.","whyItMatters":"The benchmark addresses the challenge of evaluating over-alignment in LLMs performing legitimate legal tasks. It provides a standardized way to measure refusal behavior across languages and task types, aiding in the selection and mitigation of models for sensitive translation and summarization work.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"437f0e720580a98c5e3c2868201b51df55e7b88ae14ddf635fca01c055777d3c"},"motivation":"While the wider applicability of LLMs in the legal field is currently debated due to their reliability and the gravity of any errors, narrow uses with well-understood and mitigated risks have emerged.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.23375","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_statabench_6a243141","familyId":"bmf_81f1ac51c69f","name":"StatABench","oneLine":"StatABench evaluates LLMs' statistical analysis capabilities through two components: Stat-Closed, 404 questions across 18 topics in multiple formats, and Stat-Open, 30 complex modeling tasks from competitions, scored via LLM-as-judge.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22977","pdf":"https://arxiv.org/pdf/2606.22977","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.22977"},"evidence":{"snippet":"To bridge this gap, we introduce StatABench (Statistical AnalysisBenchmark), a benchmark designed to systematically assess LLMs' statistical analysis capabilities.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22977"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"StatABench evaluates LLMs' statistical analysis capabilities through two components: Stat-Closed, 404 questions across 18 topics in multiple formats, and Stat-Open, 30 complex modeling tasks from competitions, scored via LLM-as-judge.","whyItMatters":"Addresses the need for systematic evaluation of LLMs in statistical analysis, revealing performance gaps and challenges in tool-grounded reasoning and modeling.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ace7c1855e953724bb15c82ed04e7a3c984e30bfcf14f2807724e35d5f62534f"},"motivation":"Statistical analysis is a broad, complex field requiring both domain knowledge and tool proficiency.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22977","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_rift-bench_5db6cac9","familyId":"bmf_140e541b846e","name":"RIFT-Bench","oneLine":"RIFT-Bench is described as a methodology for dynamic red-teaming of agentic AI systems, using a graph representation to enable unified evaluations across diverse agentic architectures. It operates in two phases: Discovery and Scanning, deploying adaptive adversarial attacks and producing evaluation reports.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.23927","pdf":"https://arxiv.org/pdf/2606.23927","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.23927"},"evidence":{"snippet":"To address this gap, we introduce RIFT-Bench, a graph representation-driven methodology for dynamic red-teaming that enables unified evaluations across diverse agentic architectures.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.23927"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RIFT-Bench is described as a methodology for dynamic red-teaming of agentic AI systems, using a graph representation to enable unified evaluations across diverse agentic architectures. It operates in two phases: Discovery and Scanning, deploying adaptive adversarial attacks and producing evaluation reports.","whyItMatters":"As agentic AI systems become more autonomous, they introduce new attack surfaces beyond traditional LLM vulnerabilities. RIFT-Bench aims to provide a scalable foundation for security evaluation across heterogeneous agentic architectures, which could aid in comparing the robustness of different systems and evaluating mitigation strategies.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e40cc81229cc15d976980edd192b68f9c210a046ae7072b970707b7ca3672362"},"motivation":"Agentic AI systems powered by large language models (LLMs) are rapidly evolving into autonomous decision-making systems, exposing attack vectors beyond those of traditional LLM vulnerabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.23927","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_quechuatok_31b7234f","familyId":"bmf_7dbc10d7377f","name":"QuechuaTok","oneLine":"A comparison of tokenization strategies (BPE, Unigram LM, WordPiece, PRPE) for Southern Quechua using a 200k-sentence corpus and a finite-state morphological analyzer as reference, with metrics including fertility rate, OOV rate, and morphological boundary accuracy.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.23943","pdf":"https://arxiv.org/pdf/2606.23943","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.23943"},"evidence":{"snippet":"We present QuechuaTok, a systematic benchmark comparing four tokenization strategies - BPE, Unigram LM, WordPiece, and a morphology-aware PRPE tokenizer - for Southern Quechua (quz), a low-resource agglutinative language spoken by 8-10 million people in South America.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.23943"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A comparison of tokenization strategies (BPE, Unigram LM, WordPiece, PRPE) for Southern Quechua using a 200k-sentence corpus and a finite-state morphological analyzer as reference, with metrics including fertility rate, OOV rate, and morphological boundary accuracy.","whyItMatters":"Evaluates tokenizer quality for agglutinative low-resource languages, showing that fertility rate alone is insufficient and that morphological boundary accuracy provides a more meaningful signal.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7790df887552703ec0ab8f7d6a75a3d5ebd8533a63c2dccb304189ca55d56638"},"motivation":"Tokenization is a foundational step in NLP pipelines, yet standard evaluation metrics such as fertility rate fail to capture morphological correctness for agglutinative languages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.23943","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_pivotsbench_17265409","familyId":"bmf_c9a766b802d9","name":"PIVOTSBench","oneLine":"PIVOTSBench evaluates multimodal large language models on fine-grained interpersonal relationship reasoning. It includes tasks predicting bidirectional relationship dimensions from videos and auxiliary tasks on visual cue identification.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.23092","pdf":"https://arxiv.org/pdf/2606.23092","project":"https://flynnzhangsx.github.io/PIVOTSBench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.23092"},"evidence":{"snippet":"To address this gap, we introduce PIVOTS, the first benchmark built from Social-IQ 2.0 and YouTube data to evaluate MLLMs' ability to predict bidirectional interpersonal relationship dimensions grounded in established psychology research.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.23092"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PIVOTSBench evaluates multimodal large language models on fine-grained interpersonal relationship reasoning. It includes tasks predicting bidirectional relationship dimensions from videos and auxiliary tasks on visual cue identification.","whyItMatters":"This benchmark addresses the lack of evaluation for multimodal social reasoning, providing a standardized test for model capabilities in understanding nuanced interpersonal cues, which is crucial for developing AI that interacts naturally in social contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4ad2fc3ec8eb2794d1970a9432d771d0c35c8415d44e9c90878ad0b7870409f8"},"motivation":"Humans possess an innate ability to understand fine-grained interpersonal relationships, which is central to everyday social interactions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.23092","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_motionhalluc_82d851ff","familyId":"bmf_5aa2d058a5cc","name":"MotionHalluc","oneLine":"MotionHalluc is a benchmark evaluating kinematic hallucinations in cross-video motion comparison. It includes 1540 questions over 553 video pairs, assessing directional, attributional, and temporal hallucinations in generated instructions.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.23061","pdf":"https://arxiv.org/pdf/2606.23061","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.23061"},"evidence":{"snippet":"To systematically investigate these hallucinations, we introduce MotionHalluc, a dedicated benchmark for evaluating motion hallucinations in paired-video comparison.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.23061"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MotionHalluc is a benchmark evaluating kinematic hallucinations in cross-video motion comparison. It includes 1540 questions over 553 video pairs, assessing directional, attributional, and temporal hallucinations in generated instructions.","whyItMatters":"Large multimodal models often produce motion hallucinations in paired-video comparison tasks. A systematic benchmark like MotionHalluc could help measure and reduce these errors, potentially improving the reliability of automated motion feedback systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d3f82b13758784501626b4e9040b92eb94966af01ac8e72cf835d080fa5fe9bb"},"motivation":"Motion instruction generation in cross-video comparison aims to produce corrective feedback that describes the differences between a query and a reference motion.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.23061","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_hakari-bench_2bd08ca4","familyId":"bmf_c03e69e4986c","name":"HAKARI-Bench","oneLine":"Evaluates retrieval architectures and efficiency settings (dimensionality reduction, quantization, reranking) across 35 benchmarks and 551 tasks in 43 languages, with unified conditions and metrics for BM25, dense, sparse, late interaction, and reranker models.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.IR"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22778","pdf":"https://arxiv.org/pdf/2606.22778","project":null,"code":"https://github.com/hakari-bench/hakari-bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.22778"},"evidence":{"snippet":"We present HAKARI-Bench, a lightweight benchmark that reconstructs existing retrieval suites into small datasets (Nano-sets): 35 benchmarks and 551 tasks across 43 languages in a unified format, enabling same-condition, model-agnostic comparison of five retrieval families (BM25, dense, sparse, late interaction, rerankers) and their efficiency variants.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":30,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22778"},"ranking":{"90d":{"score":66,"rank":91,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates retrieval architectures and efficiency settings (dimensionality reduction, quantization, reranking) across 35 benchmarks and 551 tasks in 43 languages, with unified conditions and metrics for BM25, dense, sparse, late interaction, and reranker models.","whyItMatters":"Fills the gap for a lightweight, high-fidelity proxy for full retrieval benchmarks, enabling rapid model selection, regression detection, and quality-efficiency trade-off analysis under consistent conditions, which is otherwise computationally prohibitive.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1a479749837c506d9b8e8f26a60193dda51557e0f8f714b4c24bdf6473759177"},"motivation":"With the rapid spread of retrieval-augmented generation and semantic search, choosing the right embedding and retrieval configuration is increasingly hard.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22778","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_geofidelity-bench_953b4352","familyId":"bmf_99a8cc850143","name":"GeoFidelity-Bench","oneLine":"GeoFidelity-Bench evaluates segment-level geographic fidelity in text-to-image street-view generation using 7,117 Mapillary images across 109 road segments in 25 cities.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.23669","pdf":"https://arxiv.org/pdf/2606.23669","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.23669"},"evidence":{"snippet":"We introduce GeoFidelity-Bench, a reference-panel benchmark for segment-conditioned geographic fidelity in street-view generation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.23669"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GeoFidelity-Bench evaluates segment-level geographic fidelity in text-to-image street-view generation using 7,117 Mapillary images across 109 road segments in 25 cities.","whyItMatters":"It addresses the gap between city-plausible and segment-accurate street-view generation, offering a protocol for comparing models on local geographic discrimination.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"49d5d910f8e12f35eec199a7cdbdc82ffa932b76c83c7e102f095d8bbe9eaea2"},"motivation":"Text-to-image models can generate visually plausible city streets, but whether their outputs correspond to a requested road segment rather than a generic city prior remains unclear.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.23669","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_enterpriseclawbench_252ea1ca","familyId":"bmf_7de792287b02","name":"EnterpriseClawBench","oneLine":"Evaluates coding agents on realistic enterprise workflows using 852 reproducible tasks with recovered fixtures, prompts, role classes, skill subclasses, hard rules, and semantic rubrics. Scoring covers artifact delivery, visual quality, cost, runtime, and skill transfer.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.23654","pdf":"https://arxiv.org/pdf/2606.23654","project":null,"code":"https://github.com/FrontisAI/EnterpriseClawBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.23654"},"evidence":{"snippet":"We introduce EnterpriseClawBench, an enterprise agent benchmark constructed from proprietary, real-world agent sessions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":80,"hfDailySubmittedAt":null,"githubStars":47,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.23654"},"ranking":{"90d":{"score":77,"rank":34,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates coding agents on realistic enterprise workflows using 852 reproducible tasks with recovered fixtures, prompts, role classes, skill subclasses, hard rules, and semantic rubrics. Scoring covers artifact delivery, visual quality, cost, runtime, and skill transfer.","whyItMatters":"Enterprise agent evaluation often collapses performance into a single score, ignoring cost, runtime, and artifact quality. This benchmark provides a multi-faceted scoring contract and a reusable construction/evaluation protocol, enabling practical comparisons of harness-model systems in real workplace settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2da25124a1691b0d43a4edc485fe703246c2b9b27ab36a755cf69301364cde2b"},"motivation":"Enterprise agents increasingly operate inside workspaces: they read heterogeneous files, invoke tools, and deliver business artifacts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.23654","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FrontisAI","organizationType":"company-research-lab","sourceUrl":"https://github.com/FrontisAI/EnterpriseClawBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_ehr-complex_08f2ee7c","familyId":"bmf_68a1189baa1d","name":"EHR-Complex","oneLine":"EHR-Complex evaluates clinical agent performance on interactive reasoning over MIMIC-IV electronic health records. It consists of about 52K tasks across six clinical intents, requiring agents to execute SQL or Python in a sandboxed environment to answer patient- and population-level queries. Scoring is based on exact-match accuracy against expected outcomes.","area":"Agents & Tool Use","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.23301","pdf":"https://arxiv.org/pdf/2606.23301","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.23301"},"evidence":{"snippet":"In this work, we introduce EHR-Complex, a large-scale benchmark designed for interactive clinical database reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.23301"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EHR-Complex evaluates clinical agent performance on interactive reasoning over MIMIC-IV electronic health records. It consists of about 52K tasks across six clinical intents, requiring agents to execute SQL or Python in a sandboxed environment to answer patient- and population-level queries. Scoring is based on exact-match accuracy against expected outcomes.","whyItMatters":"Existing clinical benchmarks often rely on simplified, static SQL generation, failing to reflect real-world EHR complexity. EHR-Complex introduces interactive, multi-step reasoning tasks with compositional queries, revealing that state-of-the-art models achieve only 62.3% accuracy and exhibit fragility under repeated sampling, highlighting significant room for improvement in robust clinical reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6a7a4bcc87590e1d877d8fb6b3ed8b222b7c1bf3f1e614ed94e9d017e0713e3c"},"motivation":"Clinical agents promise to democratize access to electronic health records (EHRs), yet existing benchmarks fail to reflect the complexity of practical EHR analysis, e.g., often operating on idealized, clean EHRs via static SQL generation rather than interactive execution.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.23301","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_dynamicmem_faf51af5","familyId":"bmf_bbff9299325a","name":"DynamicMem","oneLine":"DynamicMem evaluates long-horizon memory in LLM agents through a synthetic benchmark with 15 months of multi-app activity per user across 16 applications, including attributes, habits, and preferences that evolve and must be inferred from scattered evidence. Scoring occurs at quarterly checkpoints, tracking performance as history grows.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22877","pdf":"https://arxiv.org/pdf/2606.22877","project":"https://wenyaxie023.github.io/DynamicMem/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.22877"},"evidence":{"snippet":"We introduce DynamicMem, a synthetic benchmark that constructs 15 months of activity per user, providing long-term multi-app data that real users' privacy keeps out of reach.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22877"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DynamicMem evaluates long-horizon memory in LLM agents through a synthetic benchmark with 15 months of multi-app activity per user across 16 applications, including attributes, habits, and preferences that evolve and must be inferred from scattered evidence. Scoring occurs at quarterly checkpoints, tracking performance as history grows.","whyItMatters":"Existing memory benchmarks use short, simplified interactions, missing real-world complexity. DynamicMem provides a long-horizon, multi-app evaluation that yields insights into memory failures, such as degradation with history length and retrieval-driven errors, which can guide improvements in memory systems for personal assistants.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a12059d7f5c24799fcc4e6e7d2f7faa87292b2a9544de5f4f2694b6e6459213a"},"motivation":"LLM agents increasingly act as personal assistants that must remember a user's profile over months: who they are (attributes), what they routinely do (habits), and what they prefer (preferences), and keep it updated as jobs, routines, and tastes drift.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22877","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_callbench_09a4fc23","familyId":"bmf_502e5ff2677a","name":"CallBench","oneLine":"CallBench is a Chinese benchmark for evaluating dual-goal coordination in phone call assistants, with 50,000 multi-turn dialogues across six scenarios and a preset-aware turn-level scoring protocol.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.22635","pdf":"https://arxiv.org/pdf/2607.22635","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.22635"},"evidence":{"snippet":"We introduce \\textsc{CallBench}, a Chinese benchmark for evaluating dual-goal coordination in phone call assistants.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.22635"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CallBench is a Chinese benchmark for evaluating dual-goal coordination in phone call assistants, with 50,000 multi-turn dialogues across six scenarios and a preset-aware turn-level scoring protocol.","whyItMatters":"Existing dialogue benchmarks focus on single explicit goals, while real phone assistants must balance the owner's preset and the caller's dynamic goal. CallBench provides a reusable evaluation to measure turn-level decisions under proxy constraints.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3825108a9b4ac0a2ddb8111fde25627ed9cd0a9c8ea24fa29d5a269e26dbf0f7"},"motivation":"Target-oriented dialogue systems have demonstrated strong capabilities in completing user goals through interactive conversations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.22635","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_t-impact_38620781","familyId":"bmf_bc176ef6047b","name":"T-IMPACT","oneLine":"T-IMPACT evaluates models on severity-aware detection of manipulated news-style image-text pairs, with 98,786 examples covering pristine, image-only, text-only, and joint manipulations, alongside calibrated continuous severity scores, coarse labels, and grounding metadata.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22339","pdf":"https://arxiv.org/pdf/2606.22339","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.22339"},"evidence":{"snippet":"We introduce T-IMPACT, a first-release severity-aware benchmark for manipulated news-style image-text pairs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22339"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"T-IMPACT evaluates models on severity-aware detection of manipulated news-style image-text pairs, with 98,786 examples covering pristine, image-only, text-only, and joint manipulations, alongside calibrated continuous severity scores, coarse labels, and grounding metadata.","whyItMatters":"Existing multimodal manipulation benchmarks focus on authenticity or manipulation type, lacking graded impact severity. T-IMPACT fills this gap with a calibrated continuous severity signal, enabling evaluation of models' ability to judge contextual impact, which is essential for mitigating persuasive misinformation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f1424a71a9c6e791436ef4d9dda5c9105316a0b14e0877cf67e9ba1d6e9d9f8e"},"motivation":"Recent advances in vision-language models and generative editing systems have made it increasingly easy to produce persuasive multimodal misinformation by altering images, text, or both jointly.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22339","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_robomme-interference_7b518e1a","familyId":"bmf_f0a1facb41ae","name":"RoboMME-Interference","oneLine":"RoboMME-Interference evaluates robot long-context memory under cross-session interference. The benchmark builds on RoboMME, constructing session histories per query episode with relevant demonstration plus controlled unrelated sessions, and measures task success for memory-augmented vision-language-action models.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22338","pdf":"https://arxiv.org/pdf/2606.22338","project":"https://robotmemorybench.com","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.22338"},"evidence":{"snippet":"To measure how current robot memory systems perform on longer sessions with more distractions, we introduce RoboMME-Interference, a cross-session benchmark built on RoboMME.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22338"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"RoboMME-Interference evaluates robot long-context memory under cross-session interference. The benchmark builds on RoboMME, constructing session histories per query episode with relevant demonstration plus controlled unrelated sessions, and measures task success for memory-augmented vision-language-action models.","whyItMatters":"Existing robot memory benchmarks ignore realistic multi-session interference. RoboMME-Interference quantifies how memory decays with unrelated sessions and whether retrieval mechanisms restore robustness, providing practical decision value for long-deployed robots.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b9d6394cb1ee7992ed3105c080ada8703802ee95f9e1baebb3eaabc55a46ca8e"},"motivation":"Robots deployed in realistic settings will accumulate experience across many sessions and tasks over their deployment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22338","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"robotmemorybench.com","organizationType":"academic-lab","sourceUrl":"https://robotmemorybench.com","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_rigorbench_122c9d4f","familyId":"bmf_41aa52836b4e","name":"RigorBench","oneLine":"RigorBench evaluates autonomous AI coding agents on engineering process discipline across five pillars: Planning Fidelity, Verification Coverage, Recovery Efficiency, Abstention Quality, and Atomic Transition Integrity. It includes 30 tasks in five categories and a composite RigorScore metric.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22678","pdf":"https://arxiv.org/pdf/2606.22678","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.22678"},"evidence":{"snippet":"We introduce RigorBench, the first benchmark designed to measure process discipline in AI coding agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22678"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"RigorBench evaluates autonomous AI coding agents on engineering process discipline across five pillars: Planning Fidelity, Verification Coverage, Recovery Efficiency, Abstention Quality, and Atomic Transition Integrity. It includes 30 tasks in five categories and a composite RigorScore metric.","whyItMatters":"Existing agent benchmarks focus on outcome correctness, ignoring process quality. RigorBench fills this gap by measuring how agents plan, verify, and recover, providing a more comprehensive assessment for reliable deployment in real-world software engineering.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b307b8177198c0f9921172b8e9e4fc4c0170e57c3b449c8940f317cd19278d17"},"motivation":"Agentic coding harnesses - such as Agent-Skills, Superpowers, and Agent-Rigor - are increasingly deployed to augment underlying LLMs for real-world software engineering tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22678","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_planbench-xl_459bd32f","familyId":"bmf_023188610684","name":"PlanBench-XL","oneLine":"PlanBench-XL evaluates LLM tool-use agents on long-horizon planning in large-scale tool ecosystems. It includes 327 retail tasks across 1,665 tools, requiring iterative tool retrieval and use. Optional blocker mechanisms inject missing, failing, or distracting tools to test adaptive planning. Scoring is based on final answer accuracy and auxiliary metrics.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22388","pdf":"https://arxiv.org/pdf/2606.22388","project":null,"code":"https://github.com/JiayuJeff/PlanBench-XL","data":null,"hfPaper":"https://huggingface.co/papers/2606.22388"},"evidence":{"snippet":"To address this gap, we introduce PlanBench-XL, an interactive benchmark of 327 retail tasks over 1,665 tools that tests whether agents can iteratively retrieve usable tools, invoke them to uncover intermediate evidence for subsequent calls toward the final goal.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":96,"hfDailySubmittedAt":null,"githubStars":40,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22388"},"ranking":{"90d":{"score":76,"rank":42,"coverage":0.7,"confidence":"Medium"}},"description":"PlanBench-XL evaluates LLM tool-use agents on long-horizon planning in large-scale tool ecosystems. It includes 327 retail tasks across 1,665 tools, requiring iterative tool retrieval and use. Optional blocker mechanisms inject missing, failing, or distracting tools to test adaptive planning. Scoring is based on final answer accuracy and auxiliary metrics.","whyItMatters":"Existing benchmarks often assume full tool visibility, which underrepresents real-world agent deployment. PlanBench-XL fills this gap by testing planning under retrieval-limited visibility and tool failures, providing insight into robustness and adaptability of LLM agents in complex tool environments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cbfd3b537693765f431621cd94544bd7036ffce327d95eff0c803eff393bfa90"},"motivation":"LLM agents increasingly operate in large tool ecosystems, where real-world tasks require discovering relevant tools, inferring implicit sub-goals, and adapting to dynamic environments over long horizons.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22388","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"PlanBench-XL Team","organizationType":"academic-lab","sourceUrl":"https://github.com/JiayuJeff/PlanBench-XL","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_mmgist_394485cc","familyId":"bmf_06d36f826ac3","name":"MMGist","oneLine":"MMGist is a curated multimodal benchmark covering seven capability dimensions with 7,262 items, derived from 18 existing benchmarks via filtering pipelines. It evaluates vision-language models across dimensions such as Visual Logic and Expert Knowledge.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22437","pdf":"https://arxiv.org/pdf/2606.22437","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.22437"},"evidence":{"snippet":"To this end, we propose MMGist, a curated benchmark that covers seven capability dimensions and contains 7,262 items.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22437"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MMGist is a curated multimodal benchmark covering seven capability dimensions with 7,262 items, derived from 18 existing benchmarks via filtering pipelines. It evaluates vision-language models across dimensions such as Visual Logic and Expert Knowledge.","whyItMatters":"MMGist addresses the need for reliable and discriminative evaluation of vision-language models, aiming to reduce redundancy and saturation in existing benchmarks while preserving model rankings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e4221c16868e4b94571867d898969f7c8b4c85170f3ea03cbc2f98f43bb03e61"},"motivation":"We conduct a systematic study of 18 widely used vision-language benchmarks and identify three major issues: 1) many items do not rely on visual cues and therefore fail to effectively measure multimodal understanding; 2) many items are already close to performance saturation for current LVLMs, which limits their discriminative power; 3) a small number of anomalous items affect the reliability of evaluation results.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22437","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mapreason-osm_48426257","familyId":"bmf_f407dc71cb3b","name":"MapReason-OSM","oneLine":"MapReason-OSM evaluates vision-language models on graph-verifiable mobility decisions from self-rendered OpenStreetMap panels. It covers 12 tasks in routing, facility location, and visual disambiguation, with structured outputs scored against hidden oracles for validity, legality, optimality, and constraint satisfaction, plus cross-zoom consistency.","area":"Multimodal","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Logistics"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22597","pdf":"https://arxiv.org/pdf/2606.22597","project":null,"code":"https://github.com/Vi-Sri/mapreason-osm","data":null,"hfPaper":"https://huggingface.co/papers/2606.22597"},"evidence":{"snippet":"We present MapReason-OSM, a benchmark and evaluation harness for graph-verifiable mobility decisions on self-rendered OpenStreetMap panels.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22597"},"ranking":{"90d":{"score":29,"rank":372,"coverage":0.55,"confidence":"Low"}},"description":"MapReason-OSM evaluates vision-language models on graph-verifiable mobility decisions from self-rendered OpenStreetMap panels. It covers 12 tasks in routing, facility location, and visual disambiguation, with structured outputs scored against hidden oracles for validity, legality, optimality, and constraint satisfaction, plus cross-zoom consistency.","whyItMatters":"Existing map benchmarks often rely on free-text or multiple-choice answers that cannot be verified against road networks. This benchmark provides a reproducible, exact scoring contract for decision-making tasks, supporting comparison of VLM capabilities in logistics, delivery, and accessible navigation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"805fbe51bda3d2baa37217f3de7124299a9f0d3715ee546ff553f515c8261a57"},"motivation":"Vision-language models (VLMs) are increasingly used to read maps for logistics, delivery, and accessible navigation, where the output is an actionable decision (a route, a pin, a parking choice) that must respect the road network.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22597","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Vi-Sri","organizationType":"community","sourceUrl":"https://github.com/Vi-Sri/mapreason-osm","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_macagentbench_782bfad8","familyId":"bmf_bd6b325f56e2","name":"MacAgentBench","oneLine":"MacAgentBench evaluates computer use agents on macOS desktop automation, with 676 tasks across 25 applications, including GUI and CLI interactions. It uses deterministic rule-based evaluation and fine-grained multi-checkpoint scoring to assess sub-goal completion.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22557","pdf":"https://arxiv.org/pdf/2606.22557","project":null,"code":"https://github.com/JetAstra/MacAgentBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.22557"},"evidence":{"snippet":"We present MacAgentBench, a comprehensive macOS agent benchmark comprising 676 tasks across 25 applications, with nearly 60% involving both GUI and CLI interaction.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":48,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22557"},"ranking":{"90d":{"score":66,"rank":94,"coverage":0.7,"confidence":"Medium"}},"description":"MacAgentBench evaluates computer use agents on macOS desktop automation, with 676 tasks across 25 applications, including GUI and CLI interactions. It uses deterministic rule-based evaluation and fine-grained multi-checkpoint scoring to assess sub-goal completion.","whyItMatters":"MacAgentBench addresses the need for benchmarks that capture framework-augmented agent capabilities and partial progress on long-horizon, multi-application tasks, providing a more granular comparison for real-world desktop automation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"826975e9c350fe0c6796ec56a396a5d639dfbc794ec5598e314bad1d4492384e"},"motivation":"Computer use agents (CUAs) have advanced rapidly in desktop automation, and a growing number of users deploy CUAs such as OpenClaw on Mac Mini for always-on automation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22557","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"JetAstra","organizationType":"community","sourceUrl":"https://github.com/JetAstra/MacAgentBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cvsbench_ac22ff1a","familyId":"bmf_5b2d8ea2ca58","name":"CVSBench","oneLine":"CVSBench evaluates cross-view spatial reasoning in vision-language models using satellite-street image pairs. It includes tasks for cross-view VQA, grounding, and viewpoint identification, with 3,297 image groups, 9,468 object-level annotations, and 40,679 QA pairs.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22476","pdf":"https://arxiv.org/pdf/2606.22476","project":null,"code":null,"data":"https://huggingface.co/datasets/zlyzlyzly/CVSBench","hfPaper":"https://huggingface.co/papers/2606.22476"},"evidence":{"snippet":"Motivated by this, we introduce CVSBench, a large-scale benchmark for evaluating cross-view spatial reasoning through satellite-street pairs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":1051,"hfDatasetLikes":1},"source":{"type":"arxiv","id":"2606.22476"},"ranking":{"90d":{"score":56,"rank":159,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":8,"datasetRankPopulation":51}},"description":"CVSBench evaluates cross-view spatial reasoning in vision-language models using satellite-street image pairs. It includes tasks for cross-view VQA, grounding, and viewpoint identification, with 3,297 image groups, 9,468 object-level annotations, and 40,679 QA pairs.","whyItMatters":"CVSBench addresses the gap in evaluating VLMs' ability to reason about scenes across drastically different viewpoints, which is crucial for applications like navigation and remote sensing. It provides a systematic protocol for measuring object-level and layout consistency under viewpoint changes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"62431a688503ade3ff59c61243b231fb5e9b0ff0aa9e6c6c754b8b2e5d0a2e4a"},"motivation":"Humans can effortlessly reason about scenes across different viewpoints, yet it remains unclear whether Vision-Language Models (VLMs) possess similar cross-view spatial abilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22476","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"earth-insights","organizationType":"academic-lab","sourceUrl":"https://huggingface.co/datasets/zlyzlyzly/CVSBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_bluex-v2_e4bf2712","familyId":"bmf_c1d585d91de1","name":"BLUEX v2","oneLine":"BLUEX v2 evaluates large language models on open-ended, discursive questions from the second-phase entrance exams of UNICAMP and USP (2022–2025). The dataset includes 395 questions with 919 subquestions, covering nine subjects, with 55.7% of questions containing images represented as context-aware captions. Scoring uses an LLM-as-a-judge protocol with binary rubric criteria based on official reference answers, yielding a 0–10 score.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22723","pdf":"https://arxiv.org/pdf/2606.22723","project":null,"code":"https://github.com/TropicAI-Research/BLUEXv2","data":"https://huggingface.co/datasets/Tropic-AI/BLUEX-v2","hfPaper":"https://huggingface.co/papers/2606.22723"},"evidence":{"snippet":"In this work, we introduce BLUEX v2, a benchmark derived from the second-phase entrance exams of Brazil's two leading universities: UNICAMP (Comvest) and USP (Fuvest), spanning exam years 2022--2025.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":52,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2606.22723"},"ranking":{"90d":{"score":24,"rank":394,"coverage":1.0,"confidence":"High","datasetDownloadRank":44,"datasetRankPopulation":51}},"description":"BLUEX v2 evaluates large language models on open-ended, discursive questions from the second-phase entrance exams of UNICAMP and USP (2022–2025). The dataset includes 395 questions with 919 subquestions, covering nine subjects, with 55.7% of questions containing images represented as context-aware captions. Scoring uses an LLM-as-a-judge protocol with binary rubric criteria based on official reference answers, yielding a 0–10 score.","whyItMatters":"Portuguese-language evaluation of LLMs has been limited, especially for open-ended tasks requiring deep reasoning and generation. This benchmark provides a public, reusable testbed for assessing capabilities in mathematical reasoning, image understanding, and other dimensions, offering comparable scores across models for practical model selection.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"25b9bea1583f3f0749a1aec78df6c8536a7f11a287be1b637f2ee899196b0067"},"motivation":"Although Large Language Models (LLMs) excel in many tasks, their assessment in Portuguese has received less attention, particularly for open-ended, discursive tasks that demand deeper reasoning and generation capabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22723","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TropicAI Research","organizationType":"academic-lab","sourceUrl":"https://github.com/TropicAI-Research/BLUEXv2","role":"benchmark-publisher"}],"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_babeljudge_6d1b19e2","familyId":"bmf_625004021f92","name":"BabelJudge","oneLine":"BabelJudge audits LLM-as-a-judge reliability across languages and agent trajectories, measuring position bias, verbosity bias, order inconsistency, and cross-lingual degradation without human labels, providing a composite bias-penalised reliability score.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22329","pdf":"https://arxiv.org/pdf/2606.22329","project":null,"code":"https://github.com/Shreyaskc/BabelJudge","data":null,"hfPaper":"https://huggingface.co/papers/2606.22329"},"evidence":{"snippet":"We introduce BabelJudge, an open-source benchmark and reliability audit framework that measures all four failure modes -- position bias, verbosity bias, order inconsistency, and cross-lingual degradation -- on any judge model, without requiring human preference labels.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22329"},"ranking":{"90d":{"score":39,"rank":293,"coverage":0.55,"confidence":"Low"}},"description":"BabelJudge audits LLM-as-a-judge reliability across languages and agent trajectories, measuring position bias, verbosity bias, order inconsistency, and cross-lingual degradation without human labels, providing a composite bias-penalised reliability score.","whyItMatters":"It addresses the gap that raw accuracy hides systematic judge biases, offering a standardized method to quantify reliability for automated evaluation, crucial for trustworthy model comparisons and training data quality in multilingual and agentic contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"54e9b4bc732f37a5829e2fff89667a81889f14664841bb73afe9a6bd155c5849"},"motivation":"LLM-as-a-judge has become the dominant approach to scalable evaluation in NLP pipelines, yet judges themselves carry systematic biases that raw accuracy hides: they favor responses placed in slot A (position bias), they prefer longer responses regardless of quality (verbosity bias), and their reliability degrades sharply in lower-resource languages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22329","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_nl2scratch_fe2ce42e","familyId":"bmf_3590e32806d4","name":"NL2Scratch","oneLine":"NL2Scratch is an executable benchmark for natural-language-to-Scratch generation, consisting of 311,648 parser-valid NL-program pairs extracted from real Scratch projects. It includes a semantically validated pool of 23,594 examples and an 800-example diagnostic set. Evaluation uses Semantic Alignment Consistency (SAC), an interpretable slot-level metric for measuring semantic agreement.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Code generation"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22061","pdf":"https://arxiv.org/pdf/2606.22061","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.22061"},"evidence":{"snippet":"We introduce NL2Scratch, an executable benchmark for natural-language-to-Scratch generation comprising 311,648 parser-valid NL--program pairs, whose program side is extracted from real Scratch projects and paired with semantically aligned NL descriptions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22061"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"NL2Scratch is an executable benchmark for natural-language-to-Scratch generation, consisting of 311,648 parser-valid NL-program pairs extracted from real Scratch projects. It includes a semantically validated pool of 23,594 examples and an 800-example diagnostic set. Evaluation uses Semantic Alignment Consistency (SAC), an interpretable slot-level metric for measuring semantic agreement.","whyItMatters":"Existing NL2Code evaluation focuses on text-based languages, leaving block-based programming unevaluated. NL2Scratch enables assessment of models on event-driven, visually compositional programs, revealing gaps between lexical similarity and semantic alignment that are invisible under token-level metrics.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5f22e85182740424ca0a409b023042bb9a0624de75b20a79512b1d1dfb8e8c59"},"motivation":"Block-based programming environments such as Scratch are widely used in early programming education, yet natural-language-to-code (NL2Code) research has focused primarily on text-based languages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22061","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"NL2Scratch Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.22061","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_mat-pref_c4647f88","familyId":"bmf_fe32cc058aa2","name":"Mat-Pref","oneLine":"Mat-Pref evaluates compositional reasoning in inorganic materials via 10,837 ionic-substitution questions across 11 structure families, with splits for in-distribution performance, held-out families, and cross-property transfer.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.21830","pdf":"https://arxiv.org/pdf/2606.21830","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.21830"},"evidence":{"snippet":"We introduce Mat-Pref, a benchmark of 10,837 ionic-substitution questions across 11 inorganic structure families, grounded in density functional theory calculations from the Materials Project, with three evaluation splits that isolate in-distribution performance, generalization to entirely held-out structure families, and cross-property transfer: applying band-gap reasoning to hosts seen during training only through formation-energy supervision.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.21830"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Mat-Pref evaluates compositional reasoning in inorganic materials via 10,837 ionic-substitution questions across 11 structure families, with splits for in-distribution performance, held-out families, and cross-property transfer.","whyItMatters":"It isolates generalization types (structural transfer, property transfer, memorization) in scientific reasoning, helping to identify when RLVR improves reasoning over memorization.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3bac324eb215eab8e185a43d43fa52900eea9cd33d52800fdb374ce972795b22"},"motivation":"Reinforcement learning from verifiable rewards (RLVR) has driven rapid progress in mathematical and code reasoning, but when extended to science, existing benchmarks do not decompose what generalizes: do gains reflect structural transfer, property transfer, or memorization?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML AI4Physics 2026 Workshop","evidence":"10 pages, 4 figures, Accepted at ICML AI4Physics 2026 Workshop","evidenceUrl":"https://arxiv.org/abs/2606.21830","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML AI4Physics 2026 Workshop","reviewStatus":"accepted","decisionRaw":"10 pages, 4 figures, Accepted at ICML AI4Physics 2026 Workshop","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.21830","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"10 pages, 4 figures, Accepted at ICML AI4Physics 2026 Workshop","level":"author-claim"}]}],"publishers":[{"name":"Mat-Pref team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.21830","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_inverse-turing-bench_7691d060","familyId":"bmf_3c53c6fe0c9e","name":"Inverse Turing Bench","oneLine":"The benchmark evaluates language models on distinguishing human-only vs. human-AI multi-turn dialogues, using paired transcripts and accuracy as the metric.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.21844","pdf":"https://arxiv.org/pdf/2606.21844","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.21844"},"evidence":{"snippet":"We present Inverse Turing Bench, a benchmark that evaluates LLMs and other models on their ability to differentiate humans and AI in multi-turn text.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.21844"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"The benchmark evaluates language models on distinguishing human-only vs. human-AI multi-turn dialogues, using paired transcripts and accuracy as the metric.","whyItMatters":"It addresses the practical need for reliable human-AI differentiation in online spaces, with implications for trust and safety. The benchmark may help compare detection approaches, though its current form primarily supports the paper's findings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9b22c4ab48773d9c9bf76944e2a21876b67978012142db356ca68d7dc5fb4507"},"motivation":"As AI systems integrate into online spaces, differentiating them from humans in conversations is increasingly important.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.21844","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cfagentbench_9ccf4efc","familyId":"bmf_b402658f28ba","name":"CFAgentBench","oneLine":"CFAgentBench is an executable environment for autonomous construction-finance agents, with 1,014 task specifications across 8 domains and 77 families. A subset of 40 tasks (54 with PM extension) has oracle-validated evaluators. Grading uses state diffs, forbidden-side-effect checks, and required-output regexes, with an LLM judge only for reply quality. A public split of 711 tasks is available, and a private split of 303 is reserved for remote scoring.","area":"Agents & Tool Use","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.22000","pdf":"https://arxiv.org/pdf/2606.22000","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.22000"},"evidence":{"snippet":"We introduce CFAgentBench, a reproducible, self-hostable environment and benchmark for autonomous construction-finance agents: a CFO/controller-class agent operating across the real software stack a US construction finance team runs - ERP, project management, email, documents, pay applications, payroll, certified payroll, lien waivers, and bank/treasury portals.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.22000"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CFAgentBench is an executable environment for autonomous construction-finance agents, with 1,014 task specifications across 8 domains and 77 families. A subset of 40 tasks (54 with PM extension) has oracle-validated evaluators. Grading uses state diffs, forbidden-side-effect checks, and required-output regexes, with an LLM judge only for reply quality. A public split of 711 tasks is available, and a private split of 303 is reserved for remote scoring.","whyItMatters":"The benchmark addresses the gap in evaluating agents for finance workflows that involve real software stacks and high-stakes transactions. Its focus on functional correctness and a money-movement guard, where correct actions can fail tasks, provides a more realistic measure of deployable competence. The observed performance collapse under repeated attempts highlights the need for reliability assessment beyond single-attempt accuracy.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"714454a11dd9436cd565558f81767e20878f9b940b2058727bb699e86916ff94"},"motivation":"We introduce CFAgentBench, a reproducible, self-hostable environment and benchmark for autonomous construction-finance agents: a CFO/controller-class agent operating across the real software stack a US construction finance team runs - ERP, project management, email, documents, pay applications, payroll, certified payroll, lien waivers, and bank/treasury portals.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.22000","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"CFAgentBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.22000","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_capricorn-1k_86185f2f","familyId":"bmf_ea863771b451","name":"CapRiCorn-1K","oneLine":"CapRiCorn-1K evaluates video captioning quality and subject referential consistency across long videos (15s-10min) with audiovisual and visual-only settings. It uses LLM judge to compute accuracy, coverage, and referential consistency metrics based on manual annotations.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-20","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.21949","pdf":"https://arxiv.org/pdf/2606.21949","project":null,"code":"https://github.com/xlchen0205/CapRiCorn-1K","data":null,"hfPaper":"https://huggingface.co/papers/2606.21949"},"evidence":{"snippet":"To bridge this gap, we propose CapRiCorn-1K, a comprehensive benchmark designed to evaluate both video captioning quality and subject referential consistency across long temporal horizons and diverse video domains.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.21949"},"ranking":{"90d":{"score":25,"rank":391,"coverage":0.7,"confidence":"Medium"}},"description":"CapRiCorn-1K evaluates video captioning quality and subject referential consistency across long videos (15s-10min) with audiovisual and visual-only settings. It uses LLM judge to compute accuracy, coverage, and referential consistency metrics based on manual annotations.","whyItMatters":"Existing benchmarks focus on short videos and overall caption quality, lacking evaluation of subject referential consistency over long horizons. CapRiCorn-1K's metrics correlate with downstream understanding and generation performance, offering practical value for selecting captioning models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"41a9d510b9fcfdd080442ff8f92514749267c03bfdd2dedd97a3e59949083339"},"motivation":"Accurate and comprehensive video captions with consistent subject references are critical for downstream understanding and generation tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.21949","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"CapRiCorn-1K Team","organizationType":"academic-lab","sourceUrl":"https://github.com/xlchen0205/CapRiCorn-1K","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_r2handoversim_321897f0","familyId":"bmf_bd02714c5c81","name":"R2HandoverSim","oneLine":"Simulation benchmark for robot-to-human object handovers with standardized evaluation protocol and five complementary metrics: planning feasibility, reachability, grasp stability, grasp affordance, and safety. Compares four baseline methods on shared grasp pose prediction.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-19","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.21011","pdf":"https://arxiv.org/pdf/2606.21011","project":"https://robot-future.github.io/r2handoversim/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.21011"},"evidence":{"snippet":"We present R2HandoverSim, a simulation benchmark for robot-to-human (R2H) object handovers.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.21011"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Simulation benchmark for robot-to-human object handovers with standardized evaluation protocol and five complementary metrics: planning feasibility, reachability, grasp stability, grasp affordance, and safety. Compares four baseline methods on shared grasp pose prediction.","whyItMatters":"Addresses lack of standardized evaluation in R2H handover research, enabling objective comparison across methods. Simulation results correlate with real-world outcomes, providing a practical tool for developing and validating handover systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"69b9be08f981b88f649546214bc3f7bba2709a8a5b79a71e8d6f33324de7aa62"},"motivation":"We present R2HandoverSim, a simulation benchmark for robot-to-human (R2H) object handovers.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"2026 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS 2026)","evidence":"Accepted by the 2026 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS 2026)","evidenceUrl":"https://arxiv.org/abs/2606.21011","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"2026 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS 2026)","reviewStatus":"accepted","decisionRaw":"Accepted by the 2026 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS 2026)","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.21011","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by the 2026 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS 2026)","level":"author-claim"}]}],"publishers":[{"name":"Robot Future","organizationType":"community","sourceUrl":"https://robot-future.github.io/r2handoversim/","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_medlayxplain_c048bbf8","familyId":"bmf_befc27de9bc5","name":"MEDLAYXPLAIN","oneLine":"MedLayXPlain is a benchmark for medical lay language generation, pairing medical images with expert and lay captions across 122,789 samples from 8 imaging modalities. It introduces a 3B evaluator model scoring expert-lay alignment on five attributes.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-19","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.21194","pdf":"https://arxiv.org/pdf/2606.21194","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.21194"},"evidence":{"snippet":"To this end, we introduce MedLayXPlain, the first large-scale multimodal benchmark and evaluation framework for Medical Lay Language Generation (MLLG).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.21194"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MedLayXPlain is a benchmark for medical lay language generation, pairing medical images with expert and lay captions across 122,789 samples from 8 imaging modalities. It introduces a 3B evaluator model scoring expert-lay alignment on five attributes.","whyItMatters":"Addresses the gap between expert-level medical image descriptions and patient-accessible language, crucial for patient education and shared decision-making under recent regulations. Provides a standardized evaluation for medical VLMs in patient-facing communication.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"38408243427fa77658038347ccc33be48493991328a007a07c691952052e9496"},"motivation":"Medical Vision-Language Models (Med-VLMs) achieve strong expert-level performance, yet their ability to generate patient-accessible descriptions remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.21194","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_medhal-loc_0d281208","familyId":"bmf_fec062af73af","name":"MedHal-Loc","oneLine":"MedHal-Loc is a benchmark and metric for localization faithfulness of medical hallucination detectors, comprising a controlled subset of 300 PubMedQA-derived statements with injected span-level errors and a natural subset. It evaluates whether top-ranked error units overlap erroneous spans across four error types.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-19","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.21517","pdf":"https://arxiv.org/pdf/2606.21517","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.21517"},"evidence":{"snippet":"We introduce MedHal-Loc, a benchmark and metric for localization faithfulness -- whether a detector's top-ranked error unit actually overlaps the erroneous span.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.21517"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MedHal-Loc is a benchmark and metric for localization faithfulness of medical hallucination detectors, comprising a controlled subset of 300 PubMedQA-derived statements with injected span-level errors and a natural subset. It evaluates whether top-ranked error units overlap erroneous spans across four error types.","whyItMatters":"The benchmark addresses the evaluation gap in measuring whether hallucination detectors that claim explainability actually localize errors faithfully, providing a way to assess detection and localization validity separately.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"05d36af4258e52c03953ae111f53149123a782f12f3952eb4b1fba5cc959ab6c"},"motivation":"Detecting hallucinations in clinical text is increasingly framed as an explainability problem: systems should not merely flag an unreliable response but point to the offending span.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.21517","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_mcr-bench_10494b73","familyId":"bmf_0ade0c7988a9","name":"MCR-Bench","oneLine":"MCR-Bench is a benchmark for evaluating reproducibility in mission-critical LLM tasks, measuring output consistency across heterogeneous hardware.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-19","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.21023","pdf":"https://arxiv.org/pdf/2606.21023","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.21023"},"evidence":{"snippet":"To evaluate our approach practically, we introduce MCR-Bench, a benchmark targeting reproducibility in mission-critical tasks.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.21023"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MCR-Bench is a benchmark for evaluating reproducibility in mission-critical LLM tasks, measuring output consistency across heterogeneous hardware.","whyItMatters":"LLM deployments in finance, medicine, and law require reproducible outputs. MCR-Bench addresses the lack of standardized evaluation for numerical instability in 16-bit inference, helping practitioners select methods that balance reproducibility and performance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d0e1eb36a5bf2c868962434555e77c8adc1ca4aef192bf057ce81e714bd3b6a9"},"motivation":"As Large Language Models (LLMs) deploy into mission-critical domains (e.g., finance, medicine, and law), output reproducibility has become a strict system requirement.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.21023","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_chem2gen-bench_dc691880","familyId":"bmf_2b1bf773f6b9","name":"Chem2Gen-Bench","oneLine":"Chem2Gen-Bench evaluates chemical-to-genetic translation using 260,084 chemical and 1,099,045 genetic perturbation profiles in cell-target contexts. It measures pairwise alignment, retrieval success, and representation quality across matched perturbation settings.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Information retrieval"],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-19","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.21109","pdf":"https://arxiv.org/pdf/2606.21109","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.21109"},"evidence":{"snippet":"We introduce Chem2Gen-Bench, a benchmark comprising 260,084 chemical and 1,099,045 genetic perturbation profiles organized into cell-target contexts, and evaluate pairwise alignment, retrieval, protocol covariate associations, feature spaces, and foundation-model embeddings.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.21109"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Chem2Gen-Bench evaluates chemical-to-genetic translation using 260,084 chemical and 1,099,045 genetic perturbation profiles in cell-target contexts. It measures pairwise alignment, retrieval success, and representation quality across matched perturbation settings.","whyItMatters":"Chemical and genetic perturbations are often studied separately, leaving translation between them under-tested. This benchmark provides a comparative evaluation of retrieval and representation methods, offering decision value for selecting models that align perturbations around shared targets.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"da1053bd97aa295d773b634e1439ed27fc1b50f755fd7ef71735a0fb42842f94"},"motivation":"Virtual-cell and perturbation models are increasingly used to predict cellular responses for biomedical discovery, but chemical and genetic perturbations are not automatically interchangeable.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.21109","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"specific"},{"id":"bm_chehre_a4cc7955","familyId":"bmf_2ff6b596d710","name":"Chehre","oneLine":"Chehre is a video dataset of 2,111 facial expressions prompted by 40 emojis, with annotations transferred to synthetic faces. It defines two tasks: dominant expression recognition and distributional expression recognition, evaluating models' ability to predict human-rated labels and capture response diversity.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-19","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.21657","pdf":"https://arxiv.org/pdf/2606.21657","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.21657"},"evidence":{"snippet":"We define two benchmark tasks: dominant expression recognition, which tests whether models recover the top human-rated labels, and distributional expression recognition, which tests whether models capture the diversity of human responses.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.21657"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Chehre is a video dataset of 2,111 facial expressions prompted by 40 emojis, with annotations transferred to synthetic faces. It defines two tasks: dominant expression recognition and distributional expression recognition, evaluating models' ability to predict human-rated labels and capture response diversity.","whyItMatters":"Existing facial expression benchmarks rely on static images and basic categories, limiting evaluation of dynamic, diverse expressions. Chehre provides a controlled resource for measuring model performance on varied, distributional perception, with tasks that reveal gaps in current vision-language models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"89e7ffee0b8cf50b07733c26b83e64d5c4a00663fa5fe78fec03596c7235604c"},"motivation":"Facial expressions are nonverbal social signals used in human interaction, but facial expression recognition datasets often focus on static images, basic emotion categories, or single deterministic annotations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.21657","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Chehre Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.21657","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_aor-bench_86ee63c7","familyId":"bmf_b04fbb60d247","name":"AOR-Bench","oneLine":"AOR-Bench is a benchmark of 3,000 pseudo-harmful audio samples across six categories, designed to evaluate over-refusal in large audio language models. It assesses whether models incorrectly reject benign audio queries.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SD"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-19","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.21147","pdf":"https://arxiv.org/pdf/2606.21147","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.21147"},"evidence":{"snippet":"To study this problem, we introduce \\textbf{AOR-Bench} (\\textbf{A}udio \\textbf{O}ver-\\textbf{R}efusal \\textbf{Bench}mark), the first benchmark for over-refusal specifically designed for LALMs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.21147"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AOR-Bench is a benchmark of 3,000 pseudo-harmful audio samples across six categories, designed to evaluate over-refusal in large audio language models. It assesses whether models incorrectly reject benign audio queries.","whyItMatters":"Over-refusal in audio models is understudied. AOR-Bench provides a standardized set to measure this behavior, helping align safety mechanisms with real-world audio context.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a5a17e203a80a3d4a5cef9a08d3c84445573f948d1506a6a3925bb338ca90aff"},"motivation":"Large Audio Language Models (LALMs) have demonstrated strong performance across a wide range of audio tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.21147","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AOR-Bench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.21147","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_adamem-bench_de779711","familyId":"bmf_898bac33904d","name":"AdaMem-Bench","oneLine":"AdaMem-Bench simulates weeks of interaction with week-by-week question answering to evaluate memory policies for personalized long-horizon LLM agents. It measures QA accuracy and memory volume across different models.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-19","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.21144","pdf":"https://arxiv.org/pdf/2606.21144","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.21144"},"evidence":{"snippet":"To study this setting, we build \\textbf{AdaMem-Bench}, a benchmark that simulates weeks of interaction with week-by-week QA.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.21144"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AdaMem-Bench simulates weeks of interaction with week-by-week question answering to evaluate memory policies for personalized long-horizon LLM agents. It measures QA accuracy and memory volume across different models.","whyItMatters":"Long-term memory systems often bloat with irrelevant data. AdaMem-Bench provides a controlled environment to assess memory selection strategies, impacting efficiency and accuracy in personalized agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"292622571221fc7cc2b80aa551fadb6ec2333f2958bd85dec4230836e20d6e1c"},"motivation":"Long-term memory systems for Large Language Model (LLM) agents typically try to \\emph{remember everything}, extracting memories uniformly to retain as many facts as possible.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.21144","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AdaMem Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.21144","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_wegenbench_4908a1e1","familyId":"bmf_dddc6369e5dc","name":"WeGenBench","oneLine":"WeGenBench evaluates text-to-image generation across 4,000 bilingual prompts, with multi-dimensional tags and novel VLM-based metrics. It assesses generation quality on scene classification and specific sub-categories.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.20100","pdf":"https://arxiv.org/pdf/2606.20100","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.20100"},"evidence":{"snippet":"To address these limitations, we propose WeGenBench, a novel benchmark designed for the comprehensive, multi-perspective evaluation of text-to-image generation capabilities.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.20100"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"WeGenBench evaluates text-to-image generation across 4,000 bilingual prompts, with multi-dimensional tags and novel VLM-based metrics. It assesses generation quality on scene classification and specific sub-categories.","whyItMatters":"Existing text-to-image benchmarks lack fine-grained diagnostics. WeGenBench provides a structured resource to pinpoint model deficiencies across dimensions, guiding optimization.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c3a181f87684cd5339acae986fd52e16b1baa35686d2695c45a7d7921ceb8cee"},"motivation":"Recent text-to-image generation models have demonstrated remarkable capabilities in synthesizing highly realistic images from text inputs alone.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.20100","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"WeGenBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.20100","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_stylisticbias_ee73496d","familyId":"bmf_110092e1dedb","name":"StylisticBias","oneLine":"StylisticBias is a benchmark of 25,000 images with controlled single-attribute variations to evaluate attribute-level social bias in multimodal LLMs. It measures how visual cues shift model judgments in 25 scenarios.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.20527","pdf":"https://arxiv.org/pdf/2606.20527","project":"https://hf.co/datasets/shaghayegh/stylistic-bias-dataset","code":"https://github.com/timo-cavelius/StylisticBias","data":null,"hfPaper":"https://huggingface.co/papers/2606.20527"},"evidence":{"snippet":"We introduce StylisticBias, a controlled benchmark for evaluating attribute-level social bias in MLLMs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.20527"},"ranking":{"90d":{"score":45,"rank":232,"coverage":0.7,"confidence":"Medium"}},"description":"StylisticBias is a benchmark of 25,000 images with controlled single-attribute variations to evaluate attribute-level social bias in multimodal LLMs. It measures how visual cues shift model judgments in 25 scenarios.","whyItMatters":"Social bias in multimodal models is often confounded by identity. StylisticBias isolates visual cues, enabling targeted bias diagnosis and mitigation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ff40a8290eef1406bddf205706c66c56314b768d849515374ac802451f91aad1"},"motivation":"Multimodal large language models (MLLMs) are increasingly deployed in personally and societally consequential settings, yet the visual cues that shape how these models judge people remain poorly understood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"non-archival workshops AI4Good and Culture x AI at ICML 2026","evidence":"Accepted to the non-archival workshops AI4Good and Culture x AI at ICML 2026","evidenceUrl":"https://arxiv.org/abs/2606.20527","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"non-archival workshops AI4Good and Culture x AI at ICML 2026","reviewStatus":"accepted","decisionRaw":"Accepted to the non-archival workshops AI4Good and Culture x AI at ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.20527","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to the non-archival workshops AI4Good and Culture x AI at ICML 2026","level":"author-claim"}]}],"publishers":[{"name":"StylisticBias Team","organizationType":"academic-lab","sourceUrl":"https://github.com/timo-cavelius/StylisticBias","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_soliditybench_53feeae1","familyId":"bmf_84813a78a5db","name":"SolidityBench","oneLine":"SolidityBench contains 5,470 repository-level Solidity smart contracts with natural language descriptions, plus SolidityScore, a semantic metric emphasizing domain-critical constructs. It evaluates code generation models.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation"],"topics":["Code"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19988","pdf":"https://arxiv.org/pdf/2606.19988","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.19988"},"evidence":{"snippet":"To address this gap, we introduce SolidityBench, a benchmark of 5,470 repository-level Solidity smart contracts paired with natural language descriptions.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19988"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SolidityBench contains 5,470 repository-level Solidity smart contracts with natural language descriptions, plus SolidityScore, a semantic metric emphasizing domain-critical constructs. It evaluates code generation models.","whyItMatters":"Domain-specific code generation lacks benchmarks. SolidityBench provides a resource to measure structural and semantic correctness in high-stakes smart contracts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fa2e6184bc9af99a2cc814b3e181087b12ee77b73b2ffe7df02476ff34de2c3d"},"motivation":"Large Language Models (LLMs) have shown strong capabilities in general-purpose code generation, but their effectiveness in specialized software domains remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19988","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SolidityBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.19988","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_redact_293a3dff","familyId":"bmf_18f1255e3f73","name":"REDACT","oneLine":"REDACT is a multilingual benchmark for PII detection with 13,427 records, 51 entity types, and controlled generation axes. It allows stratified evaluation via metadata fields and includes an evaluation harness.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19881","pdf":"https://arxiv.org/pdf/2606.19881","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.19881"},"evidence":{"snippet":"We present REDACT, a systematically controlled multilingual PII benchmark with 13,427 records, 324,078 entity annotations, 51 entity types, 4,127 surface-form patterns, and 25 languages across 9 scripts.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19881"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"REDACT is a multilingual benchmark for PII detection with 13,427 records, 51 entity types, and controlled generation axes. It allows stratified evaluation via metadata fields and includes an evaluation harness.","whyItMatters":"PII detection lacks controlled benchmarks. REDACT provides systematic variation and layered evaluation to reveal failure conditions, aiding detector robustness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b0ee0792c15da051396c72af01dd24996fd7c28a84d803ee769a11951ac804f7"},"motivation":"Benchmark infrastructure for personally identifiable information (PII) detection remains limited: existing corpora cover few entity types, use ad hoc generation conditions, and do not show which surface conditions cause detector failures.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19881","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"REDACT Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.19881","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_power-systems-agent-benchmark_68561f3f","familyId":"bmf_9170ad13958e","name":"Power Systems Agent Benchmark","oneLine":"Power Systems Agent Benchmark is an executable benchmark for power-engineering agents. It includes 41 task families with deterministic evaluators that recompute engineering quantities and check constraints, plus held-out generation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.20950","pdf":"https://arxiv.org/pdf/2606.20950","project":"https://doi.org/10.5281/zenodo.20753046","code":"https://github.com/trashchenkov/power-systems-agent-benchmark","data":null,"hfPaper":"https://huggingface.co/papers/2606.20950"},"evidence":{"snippet":"We introduce the Power Systems Agent Benchmark, an executable benchmark for power-engineering agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.20950"},"ranking":{"90d":{"score":42,"rank":256,"coverage":0.7,"confidence":"Medium"}},"description":"Power Systems Agent Benchmark is an executable benchmark for power-engineering agents. It includes 41 task families with deterministic evaluators that recompute engineering quantities and check constraints, plus held-out generation.","whyItMatters":"Executable evaluation in power engineering is missing. This benchmark provides a repeatable, contamination-resistant protocol for tool-using agents, with public and hidden splits.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b76a1ac6a6646826cbebd5ce468edd53af9f5493b023d78661b8b1792ffa1be2"},"motivation":"Executable evaluation -- checking the consequences of an agent's actions with a program rather than grading its prose -- has become a prominent way to assess tool-using AI agents in software settings.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.20950","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Power Systems Agent Benchmark Team","organizationType":"academic-lab","sourceUrl":"https://github.com/trashchenkov/power-systems-agent-benchmark","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_oragentbench_12c4a136","familyId":"bmf_617a34730963","name":"ORAgentBench","oneLine":"ORAgentBench evaluates autonomous agents on end-to-end operations research tasks. Each task involves a natural-language brief, multi-file data, configuration artifacts, and a required submission schema. Agents write and run solution code, validated for schema, hard constraints, and objective quality.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19787","pdf":"https://arxiv.org/pdf/2606.19787","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.19787"},"evidence":{"snippet":"In this work, we introduce ORAgentBench, an execution-grounded benchmark for evaluating autonomous agents on challenging end-to-end operations research tasks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19787"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"ORAgentBench evaluates autonomous agents on end-to-end operations research tasks. Each task involves a natural-language brief, multi-file data, configuration artifacts, and a required submission schema. Agents write and run solution code, validated for schema, hard constraints, and objective quality.","whyItMatters":"Existing OR evaluations often decouple modeling from solving and rarely test full workflows from artifacts to validated decisions. ORAgentBench provides a realistic, execution-grounded protocol for measuring practical decision-making in operations research, helping to identify strategic weaknesses in current models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0669a1b6c4df86f9221fe089a6208124c0ce3b53ab803c950ffda48a329083c4"},"motivation":"Large language models are increasingly deployed as autonomous agents for multi-step tasks in executable environments, yet their ability to perform realistic operations research (OR) work remains unclear.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19787","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_nrt-bench_4404866b","familyId":"bmf_cb5ba2557c72","name":"NRT-Bench","oneLine":"Evaluates multi-turn red-teaming of LLM agents acting as operators of a simulated nuclear power plant control room. Safety is measured by objective loss of critical safety functions (CSFs) under adaptive attacks.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.20408","pdf":"https://arxiv.org/pdf/2606.20408","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.20408"},"evidence":{"snippet":"We present NRT-Bench, a benchmark for multi-turn red-teaming of LLM agents acting as operators of a safety-critical system, instantiated in a simulated nuclear power plant control room.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.20408"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Evaluates multi-turn red-teaming of LLM agents acting as operators of a simulated nuclear power plant control room. Safety is measured by objective loss of critical safety functions (CSFs) under adaptive attacks.","whyItMatters":"Provides a repeatable environment for assessing LLM agent robustness in safety-critical operations, where failure is objective rather than judged, and supports reproducible safety evaluation across models and defences.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"82c84e9616ac2a4127906efffb1c2ea6a54fd9ee9b316bc9198a837587256640"},"motivation":"Large language model (LLM) agents are increasingly proposed as supervisory components for safety-critical systems, yet their robustness under sustained, adaptive adversarial pressure remains poorly characterized.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.20408","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_nrityam_dcb4ad43","familyId":"bmf_9db5bbb1c2e7","name":"NRITYAM","oneLine":"A dataset of 9,260 question-answer pairs across 12 languages evaluating cultural knowledge in global dance traditions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19727","pdf":"https://arxiv.org/pdf/2606.19727","project":null,"code":"https://github.com/niladrighosh03/NRITYAM","data":null,"hfPaper":"https://huggingface.co/papers/2606.19727"},"evidence":{"snippet":"To address this gap, we present NRITYAM, a comprehensive benchmark for evaluating the cultural comprehension capabilities of language models in the context of global dance traditions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19727"},"ranking":{"90d":{"score":39,"rank":294,"coverage":0.55,"confidence":"Low"}},"description":"A dataset of 9,260 question-answer pairs across 12 languages evaluating cultural knowledge in global dance traditions.","whyItMatters":"Addresses the gap in evaluating language models' cultural comprehension, particularly for traditional performing arts, offering a multilingual resource for assessing socio-cultural understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"80cb762032c6904e90d6ad47b59da00078bbf12971795372ff80c836ad9134d3"},"motivation":"Language models have become essential tools in shaping modern workflows.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19727","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_multi-lcb_028b5b88","familyId":"bmf_d210c1214987","name":"Multi-LCB","oneLine":"Multi-LCB evaluates code generation across twelve programming languages (C++, C#, Python, Java, Rust, Go, TypeScript, JavaScript, Ruby, Kotlin, Scala, PHP) by transforming Python tasks from LiveCodeBench while preserving its contamination controls and evaluation protocol.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation"],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.20517","pdf":"https://arxiv.org/pdf/2606.20517","project":null,"code":"https://github.com/Multi-LCB/Multi-LCB","data":null,"hfPaper":"https://huggingface.co/papers/2606.20517"},"evidence":{"snippet":"We introduce Multi-LCB, a benchmark for evaluating LLMs across twelve programming languages, including Python.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":60,"hfDailySubmittedAt":null,"githubStars":28,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.20517"},"ranking":{"90d":{"score":73,"rank":57,"coverage":0.7,"confidence":"Medium"}},"description":"Multi-LCB evaluates code generation across twelve programming languages (C++, C#, Python, Java, Rust, Go, TypeScript, JavaScript, Ruby, Kotlin, Scala, PHP) by transforming Python tasks from LiveCodeBench while preserving its contamination controls and evaluation protocol.","whyItMatters":"LiveCodeBench restricted code evaluation to Python; Multi-LCB addresses the gap by enabling cross-language assessment, revealing Python overfitting and language-specific contamination in LLMs, and supporting robust multilingual code evaluation for real-world software engineering.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f648ab30e00c8861c6e20a2507217f1d4853a08596d77d744506b2efe8608d6f"},"motivation":"LiveCodeBench (LCB) has recently become a widely adopted benchmark for evaluating large language models (LLMs) on code-generation tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.20517","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Multi-LCB Team","organizationType":"academic-lab","sourceUrl":"https://github.com/Multi-LCB/Multi-LCB","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_herabench_8a4ad4a5","familyId":"bmf_c80a435ff4de","name":"HeraBench","oneLine":"HeraBench is a fault-injected benchmark for multi-device agent workflows on Linux and Android, evaluating hierarchical replanning under injected strategy- and device-level failures.","area":"Agents & Tool Use","applicationDomains":["Consumer & Productivity"],"primaryDomain":"Consumer & Productivity","industrySectors":["Consumer Technology"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.20487","pdf":"https://arxiv.org/pdf/2606.20487","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.20487"},"evidence":{"snippet":"To evaluate this capability, we introduce \\textbf{HeraBench}, a fault-injected benchmark that constructs cross-device workflows over Linux and Android devices and injects strategy- and device-level failures.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.20487"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"HeraBench is a fault-injected benchmark for multi-device agent workflows on Linux and Android, evaluating hierarchical replanning under injected strategy- and device-level failures.","whyItMatters":"Current multi-device agent benchmarks lack systematic fault injection to test recovery capabilities, making it difficult to compare hierarchical versus global replanning approaches.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"33d4b5b3d23fa2335ee9396a6fb77b16e6957b669a600505ea0b05387a0f4197"},"motivation":"Real-world computer-use tasks often span multiple applications and devices, requiring agents to coordinate heterogeneous environments under dynamic runtime failures.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.20487","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_combeval_06893a13","familyId":"bmf_1b00e4f89969","name":"CombEval","oneLine":"CombEval evaluates combinatorial counting abilities of large language models using problems generated from typed Cofola specifications, with solver-verified answers. It supports systematic variation of object types, entity scales, constraint counts, and reasoning depth.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19788","pdf":"https://arxiv.org/pdf/2606.19788","project":null,"code":"https://github.com/YuxuZhou-CN/combination-problem-generation","data":null,"hfPaper":"https://huggingface.co/papers/2606.19788"},"evidence":{"snippet":"We present CombEval, a dynamic benchmark for evaluating combinatorial counting in large language models.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19788"},"ranking":{"90d":{"score":29,"rank":373,"coverage":0.55,"confidence":"Low"}},"description":"CombEval evaluates combinatorial counting abilities of large language models using problems generated from typed Cofola specifications, with solver-verified answers. It supports systematic variation of object types, entity scales, constraint counts, and reasoning depth.","whyItMatters":"Addresses the gap in dynamic evaluation of combinatorial reasoning, providing controlled generation and exact verification to diagnose model failures in counting tasks, useful for targeted improvements.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dd040b068a53163a0e71df6f165ac27d2568013bf9d9a183de7cbdb0c838f090"},"motivation":"We present CombEval, a dynamic benchmark for evaluating combinatorial counting in large language models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19788","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"YuxuZhou-CN","organizationType":"community","sourceUrl":"https://github.com/YuxuZhou-CN/combination-problem-generation","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_bim-edit_44064775","familyId":"bmf_2a8fff911c4c","name":"BIM-Edit","oneLine":"BIM-Edit evaluates large language models on natural-language editing of Industry Foundation Classes (IFC) building models. The benchmark includes 324 editing tasks across 11 realistic building models and 36 synthetic scenes. Tasks are categorized as direct, spatial, or topological instructions, and outputs are scored on geometric accuracy, semantic validity, and topological consistency.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-18","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.20146","pdf":"https://arxiv.org/pdf/2606.20146","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.20146"},"evidence":{"snippet":"We introduce BIM-Edit, a benchmark for evaluating LLMs on natural-language editing of Building Information Models (BIM) represented in the Industry Foundation Classes (IFC) format.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.20146"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"BIM-Edit evaluates large language models on natural-language editing of Industry Foundation Classes (IFC) building models. The benchmark includes 324 editing tasks across 11 realistic building models and 36 synthetic scenes. Tasks are categorized as direct, spatial, or topological instructions, and outputs are scored on geometric accuracy, semantic validity, and topological consistency.","whyItMatters":"Construction and architectural design rely on structured BIM models; the evaluation gap is that existing benchmarks mostly test geometry and creation from scratch. BIM-Edit measures scene understanding and semantic relational preservation, providing a capability signal for practical engineering workflows where editing is central.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a0d8c53ce91c5b0295325792708c88fde7f7f4ec7fe0c3ae31b4181e646ddc89"},"motivation":"Large language models (LLMs) are increasingly applied to computer-aided design (CAD) to generate design artifacts from textual instructions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.20146","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_x-slides_283eef58","familyId":"bmf_6e62a789c14b","name":"X+Slides","oneLine":"X+Slides evaluates audience-conditioned slide generation from source documents, using a dynamic framework of 8,133 source-grounded probes across 113 topics and seven presentation scenes, reporting metrics like Audience Coverage and Efficiency.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19256","pdf":"https://arxiv.org/pdf/2606.19256","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.19256"},"evidence":{"snippet":"To bridge this gap, we introduce X+Slides, a benchmark specifically designed for audience-conditioned slide generation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19256"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"X+Slides evaluates audience-conditioned slide generation from source documents, using a dynamic framework of 8,133 source-grounded probes across 113 topics and seven presentation scenes, reporting metrics like Audience Coverage and Efficiency.","whyItMatters":"The benchmark addresses the gap in evaluating slide generation by considering target audience, offering metrics that measure coverage of audience-essential information and source grounding, which matters for selecting systems that meet specific audience needs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"907a485bfc290ad3bddfbc28c89a3d5791acf46cbd4d941410ea0e77a9c0133a"},"motivation":"Automatically generating slide decks from source documents is an important application of large language models (LLMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19256","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_worldlines_ada0e776","familyId":"bmf_38b4fbd5fcdb","name":"WorldLines","oneLine":"WorldLines evaluates long-horizon stateful embodied agents in household environments through Memory QA and Embodied Task Planning, using temporally extended household traces with dialogues and state changes.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18847","pdf":"https://arxiv.org/pdf/2606.18847","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18847"},"evidence":{"snippet":"We introduce WorldLines, a project-driven benchmark for long-horizon embodied household assistance.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":5,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18847"},"ranking":{"90d":{"score":52,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"WorldLines evaluates long-horizon stateful embodied agents in household environments through Memory QA and Embodied Task Planning, using temporally extended household traces with dialogues and state changes.","whyItMatters":"Existing benchmarks lack evaluation of long-term memory in dynamic embodied settings; WorldLines fills this gap by testing both memory retrieval and planning over extended interactions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"77c9ca12db8cf032d8beb481f9491e7bb0eedf978944ef78b14ad27ae2882300"},"motivation":"To assist humans over extended periods in real homes, embodied agents must remember user routines, world states, and past interactions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"EMNLP 2026","evidence":"Accepted to EMNLP 2026","evidenceUrl":"https://arxiv.org/abs/2606.18847","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"EMNLP 2026","reviewStatus":"accepted","decisionRaw":"Accepted to EMNLP 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.18847","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to EMNLP 2026","level":"author-claim"}]}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_therapeuticsbench_0f53fc28","familyId":"bmf_d3ec29543e47","name":"TherapeuticsBench","oneLine":"TxBench-PP evaluates AI agents on small-molecule preclinical pharmacology tasks including mechanism-of-action, pharmacodynamics, and safety reasoning, using realistic workflow snapshots and deterministic scoring.","area":"Language & Knowledge","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19245","pdf":"https://arxiv.org/pdf/2606.19245","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.19245"},"evidence":{"snippet":"We introduce TherapeuticsBench Preclinical Pharmacology (TxBench-PP), a verifiable benchmark for small-molecule preclinical pharmacology and the first focused slice of a broader TherapeuticsBench effort across drug-discovery stages and therapeutic modalities.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19245"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TxBench-PP evaluates AI agents on small-molecule preclinical pharmacology tasks including mechanism-of-action, pharmacodynamics, and safety reasoning, using realistic workflow snapshots and deterministic scoring.","whyItMatters":"Provides verifiable evaluation of agents on realistic drug discovery decisions, addressing the need for trusted benchmarks in high-stakes scientific applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5cb027a680373e4904e23518118872d33a37a614d7309b0be9862ec985e40b8b"},"motivation":"Artificial intelligence (AI) agents promise to accelerate drug discovery by compressing interpretation and decision-making loops, but practical deployment requires trusted evaluation on realistic program decisions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19245","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_textrich_83260e1b","familyId":"bmf_324a507fa21f","name":"TextRich","oneLine":"TextRich evaluates detection of AI-generated text-rich images across six categories including posters, charts, receipts, tables, and UI screenshots, with a released dataset on Hugging Face.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19259","pdf":"https://arxiv.org/pdf/2606.19259","project":null,"code":null,"data":"https://huggingface.co/datasets/Shuyiww/TextRich","hfPaper":"https://huggingface.co/papers/2606.19259"},"evidence":{"snippet":"In this paper, we introduce TextRich, a multi-domain benchmark for detecting text-rich images generated by OpenAI's GPT-Image-2.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":8434,"hfDatasetLikes":1},"source":{"type":"arxiv","id":"2606.19259"},"ranking":{"90d":{"score":60,"rank":132,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":1,"datasetRankPopulation":51}},"description":"TextRich evaluates detection of AI-generated text-rich images across six categories including posters, charts, receipts, tables, and UI screenshots, with a released dataset on Hugging Face.","whyItMatters":"Addresses the gap in detecting synthetic images with heavy text content, which is critical for digital trust and content authenticity.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"24ec9611f382fd1f8df4b61d8cc46247ee3acd4196a02b7087234a566ccefaf6"},"motivation":"Text-rich images often contain privacy-sensitive, transactional, or decision-relevant information.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19259","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_staminabench_cb49af23","familyId":"bmf_5030babf657a","name":"StaminaBench","oneLine":"StaminaBench stress-tests coding agents over 100 interaction turns of change requests, measuring how many consecutive turns they handle before failing, with programmatically generated tasks and black-box HTTP evaluation.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19613","pdf":"https://arxiv.org/pdf/2606.19613","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.19613"},"evidence":{"snippet":"We introduce StaminaBench, a benchmark that measures the stamina of coding agents: how many consecutive interaction turns (change requests) they can handle before failing.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19613"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"StaminaBench stress-tests coding agents over 100 interaction turns of change requests, measuring how many consecutive turns they handle before failing, with programmatically generated tasks and black-box HTTP evaluation.","whyItMatters":"Fills the gap in evaluating multi-turn coding agent stamina, which is critical for real-world vibe-coding sessions that often extend over many turns.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d57ee1987a79485a6b3445097afcaa1f466de6d28e5d360fb520bf31eddc373a"},"motivation":"We introduce StaminaBench, a benchmark that measures the stamina of coding agents: how many consecutive interaction turns (change requests) they can handle before failing.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19613","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_scirisk-bench_01fe7487","familyId":"bmf_700b9be2aca9","name":"SciRisk-Bench","oneLine":"SciRisk-Bench evaluates AI4Science safety across 7 disciplines and 10 risk dimensions, assessing whether models recognize and avoid risks in scientific contexts.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18936","pdf":"https://arxiv.org/pdf/2606.18936","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18936"},"evidence":{"snippet":"We introduce \\textbf{SciRisk-Bench}, a benchmark designed to evaluate AI4Science safety from two complementary perspectives: explicit risk dimensions and scientific disciplines.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18936"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SciRisk-Bench evaluates AI4Science safety across 7 disciplines and 10 risk dimensions, assessing whether models recognize and avoid risks in scientific contexts.","whyItMatters":"Existing AI4Science safety benchmarks lack explicit risk dimensions; SciRisk-Bench provides fine-grained diagnosis of safety issues across disciplines.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"69d4859f097c9b756f5ed7ded2d6ddc8fe2d2f3ba138678da77cb8fa4cf7da45"},"motivation":"Large language models (LLMs) are increasingly embedded in AI for Science (AI4Science) workflows, from scientific question answering and literature analysis to laboratory planning and autonomous discovery.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18936","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_rng-bench_1fb2eb04","familyId":"bmf_4e8d6f247af1","name":"RNG-Bench","oneLine":"RNG-Bench evaluates multimodal LLMs in controllable non-Markov games, requiring reconstruction of past observations and acting on them, with two games: Matching Pairs and 3D Maze.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19338","pdf":"https://arxiv.org/pdf/2606.19338","project":null,"code":"https://github.com/InternLM/RNGBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.19338"},"evidence":{"snippet":"We introduce RNG-Bench (Reconstructive Non-Markov Games), a benchmark suite designed to isolate a base model's ability to reconstruct past observations and act on them during multi-step interaction.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":51,"hfDailySubmittedAt":null,"githubStars":41,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19338"},"ranking":{"90d":{"score":76,"rank":44,"coverage":0.7,"confidence":"Medium"}},"description":"RNG-Bench evaluates multimodal LLMs in controllable non-Markov games, requiring reconstruction of past observations and acting on them, with two games: Matching Pairs and 3D Maze.","whyItMatters":"Addresses the gap in evaluating models' abilities to remember and act on hidden state, which is critical for real-world deployments where observations are partial.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"47e264b08d3d68b5084f6a5e801e641cd2e92b8b8cc6c9ac7434569b9803aade"},"motivation":"Deploying multimodal foundation models as closed-loop policies increasingly requires conditioning actions on observations that are no longer visible.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19338","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"InternLM","organizationType":"benchmark-organization","sourceUrl":"https://github.com/InternLM/RNGBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_rekey_04c2afe3","familyId":"bmf_e7eeb00b58dd","name":"REKEY","oneLine":"ReKey is a live benchmark protocol that regenerates visual keys in VQA images at evaluation time, creating fresh instances with new answers, to combat data leakage and memorization.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.20736","pdf":"https://arxiv.org/pdf/2606.20736","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.20736"},"evidence":{"snippet":"We propose ReKey, a live benchmark protocol that randomly regenerates the answer-bearing local detail, or visual key, in real images at evaluation time.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.20736"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ReKey is a live benchmark protocol that regenerates visual keys in VQA images at evaluation time, creating fresh instances with new answers, to combat data leakage and memorization.","whyItMatters":"Static benchmarks become contaminated over time; ReKey provides a contamination-resilient evaluation framework, ensuring scores reflect genuine visual ability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a8b033f353f7344a281de86ef91793c55376e835394eb1ab17eafb1682a8001d"},"motivation":"Static visual question answering (VQA) benchmarks age quickly: Once the items leak into training corpora, scores can reflect memorization rather than genuine visual ability, thus obscuring real progress.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.20736","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_redactionbench_13449565","familyId":"bmf_64be672a338e","name":"RedactionBench","oneLine":"RedactionBench evaluates contextual redaction of PII across 200 documents and 11 domains, with a character-level R-Score metric that treats semantically similar redactions equally.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18782","pdf":"https://arxiv.org/pdf/2606.18782","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18782"},"evidence":{"snippet":"Grounded in contextual integrity, we introduce RedactionBench, a manually annotated benchmark comprising 200 diverse documents across 11 domains, mostly seeded from real-world sources.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18782"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"RedactionBench evaluates contextual redaction of PII across 200 documents and 11 domains, with a character-level R-Score metric that treats semantically similar redactions equally.","whyItMatters":"Existing redaction benchmarks conflate extraction with privacy semantics; RedactionBench introduces contextual integrity and a metric that decouples ambiguity from precision.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bb64145d8b7ec673628d2ac2d5565c11fd1e6594793b835ac4d574d59a4818b8"},"motivation":"Large Language Models are increasingly applied to sensitive domains that require redaction of personally identifiable information (PII).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18782","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_portexto_2594bd31","familyId":"bmf_89b15b527989","name":"PorTEXTO","oneLine":"PorTEXTO evaluates visual text extraction from contemporary and culturally relevant European Portuguese (pt-PT) images. The benchmark includes synthetic and real-world samples with native-speaker-reviewed transcriptions.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19096","pdf":"https://arxiv.org/pdf/2606.19096","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.19096"},"evidence":{"snippet":"This work addresses modern OCR applications, introducing PorTEXTO, the first benchmark for contemporary and culturally relevant pt-PT visual text extraction.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19096"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"PorTEXTO evaluates visual text extraction from contemporary and culturally relevant European Portuguese (pt-PT) images. The benchmark includes synthetic and real-world samples with native-speaker-reviewed transcriptions.","whyItMatters":"European Portuguese is underrepresented in OCR benchmarks, which typically focus on high-resource languages or historical documents. PorTEXTO addresses this gap by providing a modern pt-PT evaluation set, enabling assessment of OCR models for contemporary applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0e8448455321582b00d1e3fef0df910c77cef5dd7a68988025a3100498bbf48c"},"motivation":"European Portuguese (pt-PT) is largely absent from OCR benchmarks, which skew toward high-resource languages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19096","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_physassistbench_d86f4b50","familyId":"bmf_2c7c02cd01ab","name":"PhysAssistBench","oneLine":"PhysAssistBench evaluates interactive doctor-patient-EHR assistance. It contains 1,296 physician-validated turns from MIMIC-IV cases, testing coordination of clinical knowledge, patient communication, and EHR tool use.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.18613","pdf":"https://arxiv.org/pdf/2606.18613","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18613"},"evidence":{"snippet":"We introduce PhysAssistBench, a benchmark for interactive doctor-patient-EHR assistance.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18613"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PhysAssistBench evaluates interactive doctor-patient-EHR assistance. It contains 1,296 physician-validated turns from MIMIC-IV cases, testing coordination of clinical knowledge, patient communication, and EHR tool use.","whyItMatters":"Current medical LLM evaluations isolate capabilities, but real physician assistance requires integrating knowledge, communication, and systems. PhysAssistBench provides a realistic interaction setting to assess readiness for clinical deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8d021bcb71432ace8c07aa507b4ec7027bf6656593c92cc97484ab5efc69aded"},"motivation":"The most plausible near-term role of medical LLMs is to assist rather than replace physicians, yet current evaluations often test isolated capabilities: clinical knowledge, EHR system interaction, or patient communication.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18613","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_mortarbench_63bbc02d","familyId":"bmf_765485d41ac0","name":"MortarBench","oneLine":"MortarBench evaluates LLM-based agents on mortgage loan origination tasks, covering application, underwriting, approval, and funding. It uses synthetic data with edge-case coverage and exact match accuracy scoring.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.19416","pdf":"https://arxiv.org/pdf/2606.19416","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.19416"},"evidence":{"snippet":"To fill this gap, we present MortarBench, a loan origination agent benchmark.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19416"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MortarBench evaluates LLM-based agents on mortgage loan origination tasks, covering application, underwriting, approval, and funding. It uses synthetic data with edge-case coverage and exact match accuracy scoring.","whyItMatters":"Mortgage loan origination is a critical financial process increasingly augmented by LLMs, yet no public benchmark existed. MortarBench provides a standardized evaluation to measure agent performance and identify biases in this domain.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fa0989c2db3bf72af010a4cd81d6225c9248d2df6b753754a24d8b2e3fbe61a0"},"motivation":"Loan origination is the process by which a lender creates a new loan, from application and underwriting through approval and funding.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19416","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_metabonet-bench_963d2082","familyId":"bmf_7692d167ab51","name":"MetaboNet-Bench","oneLine":"MetaboNet-Bench evaluates multimodal glucose forecasting in type 1 diabetes, using glucose, insulin, and carbohydrate data. It provides an extensible open-source evaluation framework for comparing forecasting algorithms.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.18640","pdf":"https://arxiv.org/pdf/2606.18640","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18640"},"evidence":{"snippet":"Here, we introduce MetaboNet-Bench, a benchmark for multimodal glucose forecasting for patients with type 1 diabetes that provides an extensible open-source evaluation framework for comparison of glucose forecasting algorithms that leverage glucose, insulin, and carbohydrate data.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18640"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MetaboNet-Bench evaluates multimodal glucose forecasting in type 1 diabetes, using glucose, insulin, and carbohydrate data. It provides an extensible open-source evaluation framework for comparing forecasting algorithms.","whyItMatters":"Standardized benchmarks are lacking in glucose forecasting, hindering fair comparison. MetaboNet-Bench addresses this by offering a multimodal framework, enabling assessment of algorithms that leverage multiple data signals.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3c940c0064d827a6e8a5f1287ef946a80191ee189216ca3bc3ed4fcc3acb55c9"},"motivation":"Glucose forecasting algorithms are an important aspect of glycemic control management in type 1 diabetes.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18640","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_ihbench_0d580e81","familyId":"bmf_a132a2f51250","name":"IHBench","oneLine":"IHBench evaluates post-interruption recovery in voice agents executing state-machine-driven workflows across 10 enterprise domains. It scores task fulfillment and recovery quality for six interruption types.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19595","pdf":"https://arxiv.org/pdf/2606.19595","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.19595"},"evidence":{"snippet":"We introduce IHBench (Interruption Handling Benchmark), a benchmark that evaluates post-interruption recovery in voice agents executing state-machine-driven workflows across 10 enterprise domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19595"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"IHBench evaluates post-interruption recovery in voice agents executing state-machine-driven workflows across 10 enterprise domains. It scores task fulfillment and recovery quality for six interruption types.","whyItMatters":"Voice agents must handle interruptions while maintaining workflow progress, but existing benchmarks measure only interruption timing. IHBench focuses on recovery quality, a distinct capability axis important for deployed agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"820eb257129b74e3b1d30c3a5e6d9c9f47961bfb3aba352fcbe4ac6b79eedc01"},"motivation":"Voice agents deployed in structured workflows (customer service, healthcare scheduling, account management) must handle frequent user interruptions while maintaining progress through multi-step procedures.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19595","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ht-bench_4c10005b","familyId":"bmf_712886e53afa","name":"HT-Bench","oneLine":"HT-Bench is a multi-task benchmark for tactile representation learning with egocentric vision and full-hand tactile data, including retrieval, inpainting, synthesis, and prediction tasks.","area":"Vision & 3D","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.RO"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.19161","pdf":"https://arxiv.org/pdf/2606.19161","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.19161"},"evidence":{"snippet":"To this end, we introduce \\textbf{HT-Bench}, a large-scale multi-task benchmark for dexterous full-hand tactile sensing, comprising 10M RGB frames and 7.8M tactile frames collected across 226 tasks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.19161"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"HT-Bench is a multi-task benchmark for tactile representation learning with egocentric vision and full-hand tactile data, including retrieval, inpainting, synthesis, and prediction tasks.","whyItMatters":"Tactile representation learning lacks universal benchmarks. HT-Bench explores egocentric vision paired with tactile data, providing a scalable evaluation direction for dexterous manipulation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8f8b040db355a7a4f2282ffeb970b6c4a5a58cc0dc8e02da61da86308b75c141"},"motivation":"Establishing a universal benchmark for tactile representation learning in robotic manipulation remains challenging due to the diversity of tactile sensor designs, data formats, and robot embodiments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.19161","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_gatemem_7d143ff2","familyId":"bmf_4f489248c2fa","name":"GateMem","oneLine":"GateMem evaluates memory governance in multi-principal shared-memory agents across medical, office, education, and household domains. It measures utility, access control, and active forgetting via checkpoints and a composite score.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18829","pdf":"https://arxiv.org/pdf/2606.18829","project":null,"code":"https://github.com/rzhub/GateMem","data":"https://huggingface.co/datasets/Ray368/GateMem","hfPaper":"https://huggingface.co/papers/2606.18829"},"evidence":{"snippet":"We introduce GateMem, a benchmark for multi-principal shared-memory agents.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":18,"hfDailySubmittedAt":null,"githubStars":197,"githubScope":"benchmark_repo","hfDatasetDownloads":193,"hfDatasetLikes":4},"source":{"type":"arxiv","id":"2606.18829"},"ranking":{"90d":{"score":81,"rank":17,"coverage":1.0,"confidence":"High","datasetDownloadRank":25,"datasetRankPopulation":51}},"description":"GateMem evaluates memory governance in multi-principal shared-memory agents across medical, office, education, and household domains. It measures utility, access control, and active forgetting via checkpoints and a composite score.","whyItMatters":"Shared-memory agents are understudied, yet crucial for institutional deployments. GateMem addresses the need for evaluating governance capabilities beyond simple recall, informing development of reliable multi-user agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"71def883b11a4a08b801594c7a902ee5fe8c67fd7bb7c1f56eba440b8034bfa0"},"motivation":"Memory benchmarks for LLM agents largely assume single-user settings, leaving shared assistants for hospitals, workplaces, campuses, and households understudied.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18829","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GateMem Project","organizationType":"academic-lab","sourceUrl":"https://github.com/rzhub/GateMem","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_g-idiomalign_39650ae3","familyId":"bmf_eb51932671bc","name":"G-IdiomAlign","oneLine":"G-IdiomAlign is a gloss-pivoted benchmark for cross-lingual idiom alignment, with protocols for multiple-choice equivalence and gloss-contrastive generation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18989","pdf":"https://arxiv.org/pdf/2606.18989","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18989"},"evidence":{"snippet":"We present G-IdiomAlign, a gloss-pivoted benchmark where each idiom is anchored by an English gloss from Wiktionary.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18989"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"G-IdiomAlign is a gloss-pivoted benchmark for cross-lingual idiom alignment, with protocols for multiple-choice equivalence and gloss-contrastive generation.","whyItMatters":"Idiom translation is challenging due to non-compositionality. This benchmark could support evaluation of multilingual models, but its public availability and scoring contract are not fully clear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3ab442b80666c7e46cfee103fb07c7ec581c30864e51b0fac1b83ec948ce2b69"},"motivation":"Idioms are difficult to transfer across languages due to their non-compositionality and weak surface-form grounding, making literal mappings unreliable.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ACL 2026","evidence":"Accepted to ACL 2026","evidenceUrl":"https://arxiv.org/abs/2606.18989","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ACL 2026","reviewStatus":"accepted","decisionRaw":"Accepted to ACL 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.18989","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to ACL 2026","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_forecastbench-sim_51b8b6fa","familyId":"bmf_6ae3da819bfa","name":"ForecastBench-Sim","oneLine":"A simulated-world forecasting benchmark built on Freeciv game rollouts. Evaluates probabilistic reasoning by scoring forecasts about hidden future game states, with continuous or binary questions, paired intervention worlds, and artifacts for release.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-17","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18686","pdf":"https://arxiv.org/pdf/2606.18686","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18686"},"evidence":{"snippet":"We introduce ForecastBench-Sim, a simulated-world forecasting benchmark built on game rollouts from Freeciv, a turn-based strategy game modelled on the Civilization series.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18686"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A simulated-world forecasting benchmark built on Freeciv game rollouts. Evaluates probabilistic reasoning by scoring forecasts about hidden future game states, with continuous or binary questions, paired intervention worlds, and artifacts for release.","whyItMatters":"Addresses the evaluation gap of slow real-world resolution and rare tail events by providing controllable, immediately resolvable forecasting tasks, enabling rigorous study of AI probabilistic reasoning under dynamic states.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"13c680ad7fb6de6d31f24bf5a8af02ae0f341dbae3844150f458bfd8712ba6f5"},"motivation":"Forecasting benchmarks for general-purpose AI systems usually inherit the constraints of the real world: outcomes resolve slowly, tail events are rare, and counterfactual questions are difficult to score.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18686","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_wirecraft_e44247e4","familyId":"bmf_9d7bed8a19bb","name":"WireCraft","oneLine":"WireCraft is a simulation benchmark for industrial deformable linear object manipulation, featuring three task families (connector insertion, clip routing, channel seating) with configurable difficulty, two DLO physics models, and shared evaluation metrics for RL, IL, and VLA policies across simulation and physical UR5 trajectories.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18097","pdf":"https://arxiv.org/pdf/2606.18097","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18097"},"evidence":{"snippet":"To bridge this gap, we introduce WireCraft, a simulation benchmark for industrial DLO manipulation with configurable difficulty and assets, spanning three task families: connector insertion, clip routing, and channel seating.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18097"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"WireCraft is a simulation benchmark for industrial deformable linear object manipulation, featuring three task families (connector insertion, clip routing, channel seating) with configurable difficulty, two DLO physics models, and shared evaluation metrics for RL, IL, and VLA policies across simulation and physical UR5 trajectories.","whyItMatters":"WireCraft addresses the lack of benchmarks that combine industrial fixtures, configurable tasks, and shared evaluation protocols for deformable objects, enabling reproducible comparison and progress in vision-based policy learning for industrial assembly.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"62e32d6efa59a285a09a58aa3372fddf3a04567a481033ba88cfb59363732e65"},"motivation":"Deformable Linear Objects (DLOs), such as wires and cables, are central to industrial assembly.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18097","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"WireCraft Contributors","organizationType":"benchmark-organization","sourceUrl":"https://arxiv.org/abs/2606.18097","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_ts-fault_c6d5ff11","familyId":"bmf_65f522aa76b1","name":"TS-Fault","oneLine":"Evaluates time series forecasting models under four explicit structural fault modes (time-warped shock, dependency-fracture shock, regime-transition missingness, cascading sensor-to-system failure) injected into lookback windows, with paired clean/corrupt protocol and five difficulty levels across nine datasets and six domains.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Robustness"],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18539","pdf":"https://arxiv.org/pdf/2606.18539","project":null,"code":"https://github.com/Ray-zyy/TS-Fault","data":null,"hfPaper":"https://huggingface.co/papers/2606.18539"},"evidence":{"snippet":"Treating TSF robustness as a data-quality problem, we present TS-Fault, a benchmark that evaluates forecasting models under explicit, parameterized fault scenarios with controllable semantic difficulty.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18539"},"ranking":{"90d":{"score":51,"rank":185,"coverage":0.55,"confidence":"Low"}},"description":"Evaluates time series forecasting models under four explicit structural fault modes (time-warped shock, dependency-fracture shock, regime-transition missingness, cascading sensor-to-system failure) injected into lookback windows, with paired clean/corrupt protocol and five difficulty levels across nine datasets and six domains.","whyItMatters":"Standard clean-data leaderboards assume a single error metric predicts deployed reliability, but real faults are structured events. TS-Fault provides a diagnostic protocol that isolates robustness to named fault mechanisms at tunable severities, revealing that clean accuracy anti-correlates with robustness and mechanism-level faults reorder model rankings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"845f0288b326849a82f6bea76f8d141fc12f501023298f99bc795ce8cf2906ca"},"motivation":"Time series forecasting (TSF) underpins consequential decisions in energy, transportation, finance, and healthcare, yet TSF models are almost universally ranked by a single number (e.g., average error) on clean held-out data, under the implicit assumption that it predicts deployed reliability.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18539","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HKUST(GZ)","organizationType":"academic-lab","sourceUrl":"https://github.com/Ray-zyy/TS-Fault","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_thousandworlds_3c4422be","familyId":"bmf_f43b626e717d","name":"ThousandWorlds","oneLine":"ThousandWorlds is a benchmark for climate emulation of potentially habitable exoplanets. It provides a dataset of approximately 1800 simulations from five global climate models, mapping eight planet parameters to 3D atmospheric fields. It includes three nested benchmark subsets: single-simulator regression, multi-simulator regression with complete observations, and multi-simulator regression with structured missingness. Two evaluation protocols are provided: one for ranking methods and one measuring performance relative to inter-model disagreement.","area":"Language & Knowledge","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals"],"capabilities":[],"topics":["cs.LG"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18338","pdf":"https://arxiv.org/pdf/2606.18338","project":"https://doi.org/10.57967/hf/8695","code":"https://github.com/edstevenson/ThousandWorlds","data":null,"hfPaper":"https://huggingface.co/papers/2606.18338"},"evidence":{"snippet":"We introduce ThousandWorlds, an ML-ready benchmark for exoclimate emulation and for the broader regime of low-data, multi-simulator, parameter-to-field regression.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":19,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18338"},"ranking":{"90d":{"score":63,"rank":117,"coverage":0.7,"confidence":"Medium"}},"description":"ThousandWorlds is a benchmark for climate emulation of potentially habitable exoplanets. It provides a dataset of approximately 1800 simulations from five global climate models, mapping eight planet parameters to 3D atmospheric fields. It includes three nested benchmark subsets: single-simulator regression, multi-simulator regression with complete observations, and multi-simulator regression with structured missingness. Two evaluation protocols are provided: one for ranking methods and one measuring performance relative to inter-model disagreement.","whyItMatters":"Machine-learning emulators could accelerate exoplanet climate modeling, but progress was limited by the lack of a curated multi-model dataset. This benchmark fills that gap, enabling reproducible comparison of emulation methods and addressing a regime of low-data, multi-simulator regression where deep learning underperforms.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d95e57072fd80935cd0f6c8513412e4ac080412cf8a510c50be8ac015f6d09d5"},"motivation":"The search for life beyond Earth will depend on detecting faint signatures in the atmospheres of potentially habitable exoplanets.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18338","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ThousandWorlds team","organizationType":"academic-lab","sourceUrl":"https://github.com/edstevenson/ThousandWorlds","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_sp-transientbench_52ec4ce5","familyId":"bmf_92037a40599b","name":"SP-TransientBench","oneLine":"SP-TransientBench (STB) is a real-captured multi-task benchmark for single-photon perception. It comprises 10 diverse scenes and 10,297 views captured with a solid-state single-photon LiDAR at 256×192 resolution, providing full time-of-flight histograms with multi-return behavior, calibrated camera poses, and standardized metadata. The benchmark evaluates depth estimation, multi-view reconstruction, and 3D semantic understanding, with 13-class semantic annotations for selected scenes. Dedicated data splits and evaluation protocols are provided for each task.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18952","pdf":"https://arxiv.org/pdf/2606.18952","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18952"},"evidence":{"snippet":"To bridge this gap, we introduce SP-TransientBench (STB), a real-captured multi-task benchmark for single photon perception.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18952"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SP-TransientBench (STB) is a real-captured multi-task benchmark for single-photon perception. It comprises 10 diverse scenes and 10,297 views captured with a solid-state single-photon LiDAR at 256×192 resolution, providing full time-of-flight histograms with multi-return behavior, calibrated camera poses, and standardized metadata. The benchmark evaluates depth estimation, multi-view reconstruction, and 3D semantic understanding, with 13-class semantic annotations for selected scenes. Dedicated data splits and evaluation protocols are provided for each task.","whyItMatters":"Existing single-photon perception studies rely on simulated or small-scale captures, lacking a systematic real-world evaluation. STB provides a consistent and reproducible benchmark across multiple 3D vision tasks, enabling comparative assessment of algorithms in photon-starved scenarios and advancing active 3D perception research.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b81b7540def7c0fdfd886aa4d61bcc43435ace241fc354e6e8280ecccc8fc2ea"},"motivation":"Single-photon LiDAR (SPL) based on single-photon avalanche diode (SPAD) sensing enables time-resolved photon measurements with extreme sensitivity, offering unique potential for active 3D perception in photon-starved scenarios.However, real-world single photon perception remains fundamentally challenging due to unique measurement noise and complex multi-return transient phenomena, which jointly complicate geometric…","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18952","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_safeclawbench_e7a55be8","familyId":"bmf_91d28b9baddb","name":"SafeClawBench","oneLine":"SafeClawBench is a staged benchmark for tool-using LLM agent security with 600 adversarial tasks across six attack families, reporting three endpoints: semantic attack acceptance, audit-visible harm evidence, and sandbox-observed tool/state harm.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18356","pdf":"https://arxiv.org/pdf/2606.18356","project":null,"code":null,"data":"https://huggingface.co/datasets/sairights/safeclawbench","hfPaper":"https://huggingface.co/papers/2606.18356"},"evidence":{"snippet":"We introduce SafeClawBench, a staged benchmark for tool-using agent security with 600 controlled adversarial tasks across six attack families: direct and indirect prompt injection, tool-return injection, memory poisoning, memory extraction, and ambiguity-driven unsafe inference.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":324,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2606.18356"},"ranking":{"90d":{"score":56,"rank":161,"coverage":0.3,"confidence":"Low","datasetDownloadRank":16,"datasetRankPopulation":51}},"description":"SafeClawBench is a staged benchmark for tool-using LLM agent security with 600 adversarial tasks across six attack families, reporting three endpoints: semantic attack acceptance, audit-visible harm evidence, and sandbox-observed tool/state harm.","whyItMatters":"SafeClawBench addresses the evaluation gap where existing benchmarks collapse distinct security failure stages into a single metric, making it hard to distinguish semantic compliance from actual harm. It provides separate, comparable scores across models and prompt policies, aiding in selecting agents and defenses based on the specific type of security risk.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1e8955f32543d95d9f59dfad38699b2ed8ce8a462bccecdf4748be32ffca13c4"},"motivation":"Tool-using language-model agents introduce security failures that go beyond unsafe text: they can disclose protected objects, write persistent memory, send messages, modify databases, or trigger harmful code and tool effects.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18356","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SAIRights","organizationType":"company-research-lab","sourceUrl":"https://huggingface.co/datasets/sairights/safeclawbench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_pseudobench_085d3f1c","familyId":"bmf_9abe8256366d","name":"PseudoBench","oneLine":"PseudoBench evaluates agentic auto-research systems on their ability to identify and resist pseudoscientific narratives. It contains 200 curated pseudoscientific claim-evidence pairs across five domains and assesses performance through an end-to-end research pipeline from experiment design to report writing.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.18060","pdf":"https://arxiv.org/pdf/2606.18060","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18060"},"evidence":{"snippet":"We present PseudoBench, an adversarial benchmark for evaluating whether agentic auto-research systems can identify and resist pseudoscientific narratives.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18060"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PseudoBench evaluates agentic auto-research systems on their ability to identify and resist pseudoscientific narratives. It contains 200 curated pseudoscientific claim-evidence pairs across five domains and assesses performance through an end-to-end research pipeline from experiment design to report writing.","whyItMatters":"As AI agents increasingly participate in scientific research, their susceptibility to generating plausible but misleading studies poses a risk to academic integrity. PseudoBench provides a direct measure of this risk, offering a practical tool for assessing and improving agent safety before deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8ace3faed949f12e8f49b7f8e72cce3256c8a8f2e279991aa76c886a24dfc824"},"motivation":"As Large Language Model based agents enter autonomous scientific research, their ability to resist pseudoscience becomes increasingly important.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18060","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_narrativeworldbench_d47453bd","familyId":"bmf_402c51ce9e0f","name":"NarrativeWorldBench","oneLine":"NarrativeWorldBench evaluates long-horizon narrative generation in audio drama across nine structural metrics and four Indic languages, across horizons from 10 to 200 episodes. Scoring is based on plot-beat F1 and other metrics.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.17391","pdf":"https://arxiv.org/pdf/2606.17391","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.17391"},"evidence":{"snippet":"We introduce NarrativeWorldBench, an open benchmark of nine narrative-structure metrics evaluated across horizons h in {10, 20, 50, 100, 200}, with cross-lingual evaluation across four Indic languages (Hindi, Tamil, Telugu, Marathi).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.17391"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"NarrativeWorldBench evaluates long-horizon narrative generation in audio drama across nine structural metrics and four Indic languages, across horizons from 10 to 200 episodes. Scoring is based on plot-beat F1 and other metrics.","whyItMatters":"Current LLMs degrade on long-horizon narrative coherence. This benchmark provides metrics for evaluating long-form structured content generation and cross-lingual capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"6996f6ece7ba8c94674684b0437947afc552b16fa5701464ca2ce45ce5b07017"},"motivation":"Long-form serialized audio drama, with arcs that run for 200 to 800 episodes, is a major creative medium and a setting where frontier large language models (LLMs) fail.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026 Workshops on High-dimensional Learning Dynamics (HiLD) and Culture x AI","evidence":"10 pages. Accepted to the ICML 2026 Workshops on High-dimensional Learning Dynamics (HiLD) and Culture x AI","evidenceUrl":"https://arxiv.org/abs/2606.17391","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026 Workshops on High-dimensional Learning Dynamics (HiLD) and Culture x AI","reviewStatus":"accepted","decisionRaw":"10 pages. Accepted to the ICML 2026 Workshops on High-dimensional Learning Dynamics (HiLD) and Culture x AI","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.17391","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"10 pages. Accepted to the ICML 2026 Workshops on High-dimensional Learning Dynamics (HiLD) and Culture x AI","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_longwebbench_6f8a779e","familyId":"bmf_c347b5e54e89","name":"LongWebBench","oneLine":"LongWebBench evaluates structural and functional webpage generation of long webpages. It includes 490 webpages for structural fidelity and 129 for functional interactions, using VLM-based metrics and a DOM-augmented agent pipeline.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.17727","pdf":"https://arxiv.org/pdf/2606.17727","project":null,"code":"https://github.com/zheny2751-dotcom/LongWebBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.17727"},"evidence":{"snippet":"We introduce LongWebBench, a benchmark for evaluating long-horizon webpage generation from both structural and functional perspectives.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.17727"},"ranking":{"90d":{"score":29,"rank":374,"coverage":0.55,"confidence":"Low"}},"description":"LongWebBench evaluates structural and functional webpage generation of long webpages. It includes 490 webpages for structural fidelity and 129 for functional interactions, using VLM-based metrics and a DOM-augmented agent pipeline.","whyItMatters":"Current webpage generation benchmarks focus on short static pages, missing long-horizon coherence and interactive functionality. LongWebBench provides a reusable evaluation to assess models on executable multi-step interactions, which is critical for real-world deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a89ae7f8902c620b14857b9724a4e4f640055ba7ab1d4124ceb2e6d174b62bef"},"motivation":"Recent vision-language models (VLMs) have shown promising progress in generating webpages from visual inputs, yet existing evaluations mainly focus on short, single-screen, and largely static webpages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.17727","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LongWebBench team","organizationType":"benchmark-organization","sourceUrl":"https://github.com/zheny2751-dotcom/LongWebBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_ladbench_44ed9c65","familyId":"bmf_3e35e374df54","name":"LADBench","oneLine":"LADBench evaluates large vision-language models on detecting logical anomalies in synthetic images across four domains: Residential, Urban, Collaborative, and Nature. It uses a Tiered Prompting Protocol with three progressive disclosure levels and automated scoring.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.17433","pdf":"https://arxiv.org/pdf/2606.17433","project":null,"code":null,"data":"https://huggingface.co/datasets/SahasraK/LADBench","hfPaper":"https://huggingface.co/papers/2606.17433"},"evidence":{"snippet":"To address this, we introduce LAD-bench, a benchmark of more than 1,000 curated synthetic images with logical anomalies across four domains: Residential, Urban, Collaborative, and Nature.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":82,"hfDatasetLikes":2},"source":{"type":"arxiv","id":"2606.17433"},"ranking":{"90d":{"score":46,"rank":223,"coverage":0.3,"confidence":"Low","datasetDownloadRank":32,"datasetRankPopulation":51}},"description":"LADBench evaluates large vision-language models on detecting logical anomalies in synthetic images across four domains: Residential, Urban, Collaborative, and Nature. It uses a Tiered Prompting Protocol with three progressive disclosure levels and automated scoring.","whyItMatters":"Existing anomaly benchmarks focus on visual errors, not the physical and social common sense required for open-world deployment. LADBench quantifies how much explicit assistance models need to localize and reason about logical faults, addressing a gap in evaluating sequential multimodal reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f25d55dd9bb8f8885b5281bc5e8cf2438f1c259ac5418c1a484844da40b2d5fe"},"motivation":"Large Vision Language Models (VLMs) excel at visual question answering and semantic grounding, but their capacity for autonomous logical reasoning remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"IEEE International Conference on Development and Learning (ICDL 2026)","evidence":"Accepted to the IEEE International Conference on Development and Learning (ICDL 2026)","evidenceUrl":"https://arxiv.org/abs/2606.17433","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"IEEE International Conference on Development and Learning (ICDL 2026)","reviewStatus":"accepted","decisionRaw":"Accepted to the IEEE International Conference on Development and Learning (ICDL 2026)","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.17433","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to the IEEE International Conference on Development and Learning (ICDL 2026)","level":"author-claim"}]}],"publishers":[{"name":"LADBench Team","organizationType":"academic-lab","sourceUrl":"https://huggingface.co/datasets/SahasraK/LADBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_humanoidarena_033240fc","familyId":"bmf_0b58750af400","name":"HumanoidArena","oneLine":"HumanoidArena is a simulation benchmark for egocentric hierarchical whole-body learning, evaluating high-level policies that predict whole-body actions for low-level general motion trackers across seven leg-critical human-object and human-scene interaction tasks, with perturbation-conditioned and GMT-conditioned evaluation.","area":"Robotics & Embodied AI","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.RO"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.17833","pdf":"https://arxiv.org/pdf/2606.17833","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.17833"},"evidence":{"snippet":"We introduce HumanoidArena, a simulation-first benchmark for egocentric hierarchical whole-body learning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":18,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.17833"},"ranking":{"90d":{"score":55,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"HumanoidArena is a simulation benchmark for egocentric hierarchical whole-body learning, evaluating high-level policies that predict whole-body actions for low-level general motion trackers across seven leg-critical human-object and human-scene interaction tasks, with perturbation-conditioned and GMT-conditioned evaluation.","whyItMatters":"Existing benchmarks rarely evaluate the policy-tracker interface itself, leaving open whether intermediate whole-body actions are executable, robust under task distribution shifts, and transferable across different GMT backends. HumanoidArena addresses this gap by emphasizing leg-critical interactions and transferable intermediate action representations, providing a basis for comparing hierarchical control architectures.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"95278c9da6f9ab9bedbb13e4d0f0fd5034f758a3253809662f4a4e1d9f1b0cea"},"motivation":"Humanoid robots promise whole-body interaction in human-centered environments, but scalable policy learning remains difficult because task-level decision-making and whole-body dynamic execution are tightly coupled.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.17833","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"general"},{"id":"bm_erqa-plus_148e4067","familyId":"bmf_43898951b3ad","name":"ERQA-Plus","oneLine":"ERQA-Plus evaluates embodied reasoning in AI systems with 1,766 question-answer instances grounded in 711 robot-centric images, organized by a taxonomy covering perceptual, action-centric, social-interaction, navigation-environmental, and commonsense reasoning. Scoring uses overall accuracy and SBERT similarity.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.17639","pdf":"https://arxiv.org/pdf/2606.17639","project":null,"code":"https://github.com/LUNAProject22/erqa-plus","data":"https://huggingface.co/datasets/huggingdas/erqa-plus","hfPaper":"https://huggingface.co/papers/2606.17639"},"evidence":{"snippet":"We present ERQA-Plus, a diagnostic benchmark for reasoning in embodied AI.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":"benchmark_repo","hfDatasetDownloads":229,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2606.17639"},"ranking":{"90d":{"score":52,"rank":182,"coverage":0.3,"confidence":"Low","datasetDownloadRank":23,"datasetRankPopulation":51}},"description":"ERQA-Plus evaluates embodied reasoning in AI systems with 1,766 question-answer instances grounded in 711 robot-centric images, organized by a taxonomy covering perceptual, action-centric, social-interaction, navigation-environmental, and commonsense reasoning. Scoring uses overall accuracy and SBERT similarity.","whyItMatters":"Existing visual and embodied QA benchmarks often lack control over reasoning dependencies, making it hard to distinguish genuine embodied reasoning from shortcut-driven pattern matching. ERQA-Plus provides a fine-grained diagnostic to identify strengths and weaknesses in specific reasoning categories, informing model development and deployment decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7791fd2963a0f20915e4425ee5ea4b82d7d2e57de5e196da5494206fd71fdd8e"},"motivation":"Generalist embodied agents require more than object recognition: they must reason about spatial relations, actions, procedures, human intentions, environmental constraints, and commonsense consequences from situated visual observations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.17639","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LUNAProject22","organizationType":"community","sourceUrl":"https://github.com/LUNAProject22/erqa-plus","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_ecomagentbench_d7694649","familyId":"bmf_7733964f2cda","name":"EComAgentBench","oneLine":"EComAgentBench evaluates LLM-based shopping agents on 662 long-horizon product selection tasks built from real Amazon data. Each task requires uncovering hidden requirements spread across an explicit query, tool-gated profile, and scripted clarification, verifying candidates, and committing to one product within 100 tool calls. Scoring uses typed, source-tagged rubrics for each requirement.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.17698","pdf":"https://arxiv.org/pdf/2606.17698","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.17698"},"evidence":{"snippet":"To address this gap, we introduce EComAgentBench, a benchmark of 662 tasks grounded in real Amazon products and reviews.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.17698"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EComAgentBench evaluates LLM-based shopping agents on 662 long-horizon product selection tasks built from real Amazon data. Each task requires uncovering hidden requirements spread across an explicit query, tool-gated profile, and scripted clarification, verifying candidates, and committing to one product within 100 tool calls. Scoring uses typed, source-tagged rubrics for each requirement.","whyItMatters":"Existing shopping benchmarks reveal full intent upfront, failing to reflect real-world requirements that emerge over time. EComAgentBench measures agents' ability to handle long-horizon interactions, providing a reproducible foundation for comparing dependable shopping assistance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c4d3fc0fd3c1488e9eda11c19090312cb3837eb2a1ce41843f9c0eb38eed9261"},"motivation":"As LLM-based shopping agents enter production, existing benchmarks fail to capture how a shopper's requirements arrive: stated implicitly in the query, recorded in a profile, or revealed only when the right question is asked.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.17698","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_ebench_35561bef","familyId":"bmf_e19c9d617150","name":"EBench","oneLine":"EBench is a simulation benchmark for diagnosing generalist mobile manipulation policies. It comprises 26 manipulation tasks annotated along five capability dimensions (scene, atomic skill, horizon, precision, mobility) and four generalization dimensions (object, background, instruction, mixed), with strict train/test splits and held-out online evaluation.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18239","pdf":"https://arxiv.org/pdf/2606.18239","project":null,"code":"https://github.com/InternRobotics/EBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.18239"},"evidence":{"snippet":"We present EBench, a simulation benchmark that diagnoses generalist mobile manipulation policies beyond a single success-rate scalar.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":16,"hfDailySubmittedAt":null,"githubStars":130,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18239"},"ranking":{"90d":{"score":79,"rank":25,"coverage":0.7,"confidence":"Medium"}},"description":"EBench is a simulation benchmark for diagnosing generalist mobile manipulation policies. It comprises 26 manipulation tasks annotated along five capability dimensions (scene, atomic skill, horizon, precision, mobility) and four generalization dimensions (object, background, instruction, mixed), with strict train/test splits and held-out online evaluation.","whyItMatters":"EBench addresses the need for multi-axis diagnostic evaluation of generalist manipulation models, moving beyond single success-rate metrics. It provides practical decision value by revealing capability and generalization profiles that guide model iteration and selection.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2b36e4ea16e26ac2f96a4b76eeeb920cb7cc763be748ef428da62af36d77d8c5"},"motivation":"We present EBench, a simulation benchmark that diagnoses generalist mobile manipulation policies beyond a single success-rate scalar.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18239","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Shanghai AI Laboratory","organizationType":"company-research-lab","sourceUrl":"https://github.com/InternRobotics/EBench","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_drflow_2f7bee63","familyId":"bmf_c4ede3eb8785","name":"DRFLOW","oneLine":"DRFLOW evaluates an agent's ability to predict personalized workflows, sequences of action-steps, from heterogeneous sources across five domains. It includes 100 tasks with reference workflow steps and multiple diagnostic metrics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18191","pdf":"https://arxiv.org/pdf/2606.18191","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18191"},"evidence":{"snippet":"Therefore, we introduce DRFLOW, a benchmark for evaluating personalized workflows predicted by agents from heterogeneous sources.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18191"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"DRFLOW evaluates an agent's ability to predict personalized workflows, sequences of action-steps, from heterogeneous sources across five domains. It includes 100 tasks with reference workflow steps and multiple diagnostic metrics.","whyItMatters":"Deep research systems are typically evaluated on report generation, but enterprise tasks often require actionable workflows. DRFLOW addresses this gap by assessing workflow prediction, offering metrics for factual grounding, step recovery, and personalization, which can guide development of more practical agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"508b75b001ccec7896909217fa938ce1b4edaef5dec2911a20804095a69eed10"},"motivation":"Deep research (DR) systems are increasingly used for complex information-seeking tasks, but existing works mainly focus on generating reports and summaries.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18191","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_compskillbench_264db8aa","familyId":"bmf_6de15eac705a","name":"CompSkillBench","oneLine":"CompSkillBench evaluates compositional skill routing for LLM agents: given a user query and a library of 2,209 real MCP server skills across 24 categories, systems must decompose the query into sub-tasks, retrieve a skill per sub-task, and produce an executable plan. Scoring uses step-level category recall and decomposition accuracy.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18051","pdf":"https://arxiv.org/pdf/2606.18051","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.18051"},"evidence":{"snippet":"To support evaluation, we introduce CompSkillBench, a benchmark of 300 compositional queries over 2,209 real MCP server skills spanning 24 functional categories, sourced from the public MCP ecosystem.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18051"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"CompSkillBench evaluates compositional skill routing for LLM agents: given a user query and a library of 2,209 real MCP server skills across 24 categories, systems must decompose the query into sub-tasks, retrieve a skill per sub-task, and produce an executable plan. Scoring uses step-level category recall and decomposition accuracy.","whyItMatters":"Real-world agent tasks often require composing multiple tools, but existing benchmarks emphasize single-skill selection. CompSkillBench provides a reusable, ecosystem-grounded benchmark to measure decomposition quality and retrieval in a compositional setting, helping developers identify bottlenecks in agent pipelines—particularly the critical role of task decomposition before retrieval.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"553c75df1ad6e3823e7c6253733a0e05ffa540f0c43794b319282f8b7d7dbac3"},"motivation":"LLM agents increasingly rely on external skills -- reusable tool specifications -- but real-world tasks often require composing multiple skills, not just selecting one.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18051","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SkillWeaver Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.18051","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_chlogic_3b9dd8db","familyId":"bmf_760dd213a7b3","name":"ChLogic","oneLine":"ChLogic is an English–Chinese aligned benchmark for evaluating logical reasoning robustness across surface realizations. It includes 3,000 general, 2,000 difficult, and 1,500 Chinese-only items derived from formal logical templates, with each aligned item pairing one English reference with five Chinese variants.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Robustness"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.17905","pdf":"https://arxiv.org/pdf/2606.17905","project":null,"code":"https://github.com/0328zpx/ChLogic","data":null,"hfPaper":"https://huggingface.co/papers/2606.17905"},"evidence":{"snippet":"We introduce ChLogic, an English--Chinese aligned benchmark that tests whether models preserve logical reasoning performance when the same latent logical structure is expressed in English and diverse Chinese surface realizations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.17905"},"ranking":{"90d":{"score":42,"rank":261,"coverage":0.7,"confidence":"Medium"}},"description":"ChLogic is an English–Chinese aligned benchmark for evaluating logical reasoning robustness across surface realizations. It includes 3,000 general, 2,000 difficult, and 1,500 Chinese-only items derived from formal logical templates, with each aligned item pairing one English reference with five Chinese variants.","whyItMatters":"Existing logical reasoning benchmarks focus on English, so it is unclear whether models retain performance when the same logical structure is expressed in Chinese. ChLogic provides a cross-lingual stress test, helping identify language-specific gaps and translation artifacts that affect multilingual reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d65f67ca676a6bafd2cc6a30259edab1bf36a755ab003b5a50f60c1c4e3b6182"},"motivation":"Large language models perform increasingly well on standardized logical reasoning benchmarks, but whether this ability remains robust beyond English is unclear.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.17905","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ChLogic team","organizationType":"academic-lab","sourceUrl":"https://github.com/0328zpx/ChLogic","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_ceo-bench_8522cfe6","familyId":"bmf_a0f9a9f93e4d","name":"CEO-Bench","oneLine":"CEO-Bench evaluates LLM agents on strategic resource allocation in multi-round organizational simulations with conflicting advisor inputs.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.17459","pdf":"https://arxiv.org/pdf/2606.17459","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.17459"},"evidence":{"snippet":"We introduce \\textsc{CEO-Bench}, a multi-agent benchmark that evaluates LLMs on CEO-level strategic resource reallocation -- the process of redirecting capital across business units in a multi-round, constraint-rich organizational environment.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.17459"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CEO-Bench evaluates LLM agents on strategic resource allocation in multi-round organizational simulations with conflicting advisor inputs.","whyItMatters":"It probes executive decision-making capabilities beyond isolated cognitive tasks, revealing tradeoffs in agent behavior.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ea7bd26324aaef54d6f5707b8237bae89164ce5c426f4d18250dae4f1ea69759"},"motivation":"Evaluating the decision-making capabilities of large language models (LLMs) is a growing research priority, yet existing benchmarks focus on isolated cognitive tasks such as reasoning, knowledge retrieval, and economic rationality in stylized settings.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.17459","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_ceo-bench_5cae7f96","familyId":"bmf_a0f9a9f93e4d","name":"CEO-Bench","oneLine":"CEO-Bench evaluates long-horizon agent capabilities by simulating a startup over 500 days. Agents manage pricing, marketing, budgeting, and other business aspects through a programmable Python interface, facing noisy data and changing market conditions. Performance is measured by final company balance against a rule-based baseline.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.18543","pdf":"https://arxiv.org/pdf/2606.18543","project":null,"code":"https://github.com/zlab-princeton/ceobench-src","data":null,"hfPaper":"https://huggingface.co/papers/2606.18543"},"evidence":{"snippet":"We introduce CEO-Bench, which evaluates these capabilities together by simulating a representative real-world task: operating a startup for 500 days.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":null,"githubStars":71,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.18543"},"ranking":{"90d":{"score":76,"rank":41,"coverage":0.7,"confidence":"Medium"}},"description":"CEO-Bench evaluates long-horizon agent capabilities by simulating a startup over 500 days. Agents manage pricing, marketing, budgeting, and other business aspects through a programmable Python interface, facing noisy data and changing market conditions. Performance is measured by final company balance against a rule-based baseline.","whyItMatters":"CEO-Bench addresses the evaluation gap for agents that must sustain adaptive progress over long horizons, combining uncertainty, information acquisition, and multi-step coordination. It provides a decision-useful benchmark for comparing models on realistic, dynamic business management tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1b6ad59d53ae5b54238321899dfddab40ec687f0a7d5af43c768e1809f4a5b13"},"motivation":"Language model agents are becoming proficient executors at isolated, short-horizon tasks such as software engineering and customer service.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.18543","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Princeton University","organizationType":"academic-lab","sourceUrl":"https://github.com/zlab-princeton/ceobench-src","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_aise-bench_0ef6ecf5","familyId":"bmf_9462adb5901c","name":"AISE-Bench","oneLine":"AISE-Bench is a benchmark for evaluating multi-step API-using LLM agents in information seeking on academic knowledge graphs. It comprises 1,133 QA pairs with API trajectories, validated parameters, and grounded answers, and evaluates answer quality, reference grounding, API-planning correctness, and execution success.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20498","pdf":"https://arxiv.org/pdf/2607.20498","project":"https://aise-bench.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20498"},"evidence":{"snippet":"We introduce AISE-Bench, a real-world, full-cycle annotated benchmark for information seeking on academic knowledge graphs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20498"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"AISE-Bench is a benchmark for evaluating multi-step API-using LLM agents in information seeking on academic knowledge graphs. It comprises 1,133 QA pairs with API trajectories, validated parameters, and grounded answers, and evaluates answer quality, reference grounding, API-planning correctness, and execution success.","whyItMatters":"Existing benchmarks for tool-using agents on academic graphs rely on synthetic or narrow tasks. AISE-Bench addresses this gap with real-world, full-cycle annotated data, enabling quantitative assessment of stepwise correctness, grounded summarization, and traceable reasoning in complex API workflows. It provides a challenging testbed for improving agent reliability in realistic information-seeking scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f2c2746249a7d4f2ece9a449573b9de47792cbb8c3d25e47f8b271e7cf14094e"},"motivation":"Large language models (LLMs) augmented with tools are emerging as autonomous agents capable of using Web engine, APIs, and code to solve complex, long-horizon tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"publication_reported","venue":"Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD '26), August 09-13, 2026, Jeju Island, Republic of Korea","evidence":"Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD '26), August 09-13, 2026, Jeju Island, Republic of Korea","evidenceUrl":"https://arxiv.org/abs/2607.20498","source":"arxiv-journal-reference","evidenceLevel":"strong-author-metadata","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publications":[{"venueName":"Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD '26), August 09-13, 2026, Jeju Island, Republic of Korea","publicationStatus":"published","evidence":[{"sourceType":"arxiv-journal-reference","sourceUrl":"https://arxiv.org/abs/2607.20498","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD '26), August 09-13, 2026, Jeju Island, Republic of Korea","level":"strong-author-metadata"}]}],"publishers":[{"name":"AISE-Bench Team","organizationType":"academic-lab","sourceUrl":"https://aise-bench.github.io/","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_agentmembench_9938dbed","familyId":"bmf_9d9e93f978b2","name":"AgentMemBench","oneLine":"AgentMemBench evaluates five long-term memory management strategies for conversational AI agents across three public datasets (LoCoMo, MultiDoc2Dial, MSC), covering multi-session dialogue, document grounding, and persona-grounded chat. Scoring uses retrieval metrics (Recall@k, MRR, nDCG@k), Answer F1, LLM-judge faithfulness, memory footprint, and latency over 491 annotated question turns.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-16","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.00009","pdf":"https://arxiv.org/pdf/2608.00009","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.00009"},"evidence":{"snippet":"We present AgentMemBench, a unified, reproducible benchmark evaluating five memory management strategies under identical conditions: in-context windowing (ICW), external key-value store (EKV), graph-based episodic memory (GEM), compression-based summarisation (CBS), and web-augmented memory (WAM).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.00009"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AgentMemBench evaluates five long-term memory management strategies for conversational AI agents across three public datasets (LoCoMo, MultiDoc2Dial, MSC), covering multi-session dialogue, document grounding, and persona-grounded chat. Scoring uses retrieval metrics (Recall@k, MRR, nDCG@k), Answer F1, LLM-judge faithfulness, memory footprint, and latency over 491 annotated question turns.","whyItMatters":"Long-term memory is a key bottleneck for conversational agents. AgentMemBench provides a controlled, reproducible comparison of memory strategies under identical conditions, enabling practitioners to make informed trade-offs between recall quality, accuracy, and resource cost.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"106686e4a9d95585a33c4ef011227553ef30c929aa78e88f7784e6e26e67e0b9"},"motivation":"Long-term memory remains a critical bottleneck for conversational AI agents, whose finite context windows cannot support coherent recall across thousands of turns.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.00009","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AgentMemBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2608.00009","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_uxbench_0fa77b90","familyId":"bmf_8d180072bdf4","name":"UXBench","oneLine":"UXBench evaluates LLM-generated UX critiques through local web fixtures, coverage-gated exploration, and a downstream repair agent, measuring report actionability across seven rubric dimensions.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.16262","pdf":"https://arxiv.org/pdf/2606.16262","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.16262"},"evidence":{"snippet":"We introduce UXBench, a benchmark for evaluating LLMs as interaction-grounded UX judges.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16262"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"UXBench evaluates LLM-generated UX critiques through local web fixtures, coverage-gated exploration, and a downstream repair agent, measuring report actionability across seven rubric dimensions.","whyItMatters":"There is a need for a controlled evaluation of UX critique reliability and actionability across product surfaces, but UXBench currently lacks a public release path or ongoing scoring service, limiting its standalone comparison value.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c63b53bd30985886c51073e1def18aa13779f17349489f89d391f905ad7fb9c7"},"motivation":"Large language models (LLMs) are increasingly deployed as UX judges that inspect interfaces, diagnose usability problems, and propose repairs.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.16262","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_timevista_65f9c096","familyId":"bmf_4f58296bba2d","name":"TimeVista","oneLine":"TimeVista is a benchmark for evaluating time series forecasting using Vision-Language Models (VLMs) as judges, with 5563 samples and rubrics for micro- and macro-level judgments.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.16173","pdf":"https://arxiv.org/pdf/2606.16173","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.16173"},"evidence":{"snippet":"To this end, we introduce TimeVista, a comprehensive VLM-as-a-Judge benchmark comprising 5563 time series samples paired with detailed evaluation rubrics.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16173"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TimeVista is a benchmark for evaluating time series forecasting using Vision-Language Models (VLMs) as judges, with 5563 samples and rubrics for micro- and macro-level judgments.","whyItMatters":"Addresses limitations of point-wise metrics in time series forecasting, offering a human-aligned evaluation approach that could guide model selection and improvement.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bc73f6da273b03dcc167394a747325e842410c81db3c0eb900d934dd52de6652"},"motivation":"High-quality time series forecasting is pivotal for real-world decision-making.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.16173","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_speechdx_58f3f420","familyId":"bmf_89f2b89c21d5","name":"SpeechDx","oneLine":"SpeechDx evaluates clinical speech AI across 12 datasets and 27 tasks covering various conditions, structured by speech production stages. Scoring includes classification accuracy and zero-shot transfer performance.","area":"Speech & Audio","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.17339","pdf":"https://arxiv.org/pdf/2606.17339","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.17339"},"evidence":{"snippet":"We introduce SpeechDx, a large-scale benchmark for clinical speech AI spanning 12 datasets and 27 tasks across diverse health conditions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.17339"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SpeechDx evaluates clinical speech AI across 12 datasets and 27 tasks covering various conditions, structured by speech production stages. Scoring includes classification accuracy and zero-shot transfer performance.","whyItMatters":"Clinical speech AI lacks comparable evaluation across conditions. SpeechDx provides a shared framework to assess generalization and representation quality, guiding progress toward general-purpose models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"30c3333abd92784b111d6a5e52d7e8d60f57e64abe28e8e7f357519f7fb90be4"},"motivation":"Speech offers a uniquely informative window into health by simultaneously engaging neurological, motor, respiratory, and vocal systems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.17339","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_skillchain-gym_68720c0a","familyId":"bmf_b51f64778801","name":"SkillChain-Gym","oneLine":"SkillChain-Gym evaluates production-inventory control policies with reskilling dynamics, including skill certification, forgetting, and training constraints. Metrics cover operations, resilience, capability growth, and training access across seeded disruption scenarios.","area":"Language & Knowledge","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Logistics"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.17266","pdf":"https://arxiv.org/pdf/2606.17266","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.17266"},"evidence":{"snippet":"We introduce SkillChain-Gym, a benchmark specification for reskilling-aware production-inventory control: a single-site environment with stylized worker skill-state dynamics, hard threshold certification, forgetting, and capacity-consuming training actions constrained by the same per-worker time budget as production.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.17266"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SkillChain-Gym evaluates production-inventory control policies with reskilling dynamics, including skill certification, forgetting, and training constraints. Metrics cover operations, resilience, capability growth, and training access across seeded disruption scenarios.","whyItMatters":"Workforce skills are often overlooked in production benchmarks. SkillChain-Gym provides a testbed to evaluate adaptive policies under skill dynamics, aiding in workforce planning and operational resilience.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"d9ad4f9e8e34cfcee34be6687be53b2988baec92e3e6338d8ebbd1a5984c72df"},"motivation":"Production planning increasingly has to treat workforce capability as a decision variable: certifications lapse when skills are not maintained, new products require skills the current workforce does not hold, and reskilling competes for the same worker hours needed for production.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.17266","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_sidewalkbench_6d63d9b3","familyId":"bmf_8c660e63162e","name":"SidewalkBench","oneLine":"SidewalkBench evaluates visual navigation models on urban sidewalks using GPU-accelerated simulation with procedurally generated and real-world scanned scenes, including pedestrian-reactive and long-horizon scenarios.","area":"Robotics & Embodied AI","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.RO"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.16953","pdf":"https://arxiv.org/pdf/2606.16953","project":"https://vail-ucla.github.io/SidewalkBench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.16953"},"evidence":{"snippet":"To bridge this gap, we propose SidewalkBench, a comprehensive benchmark designed for visual navigation on urban sidewalks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16953"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SidewalkBench evaluates visual navigation models on urban sidewalks using GPU-accelerated simulation with procedurally generated and real-world scanned scenes, including pedestrian-reactive and long-horizon scenarios.","whyItMatters":"It addresses the lack of a unified benchmark for sidewalk navigation, providing a standardized simulation environment to compare model performance under realistic conditions, aiding progress in visual navigation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2e3985b22c49b0e6bbf9f5d0904396ad8408508ab9643fdbac56def25361b6b5"},"motivation":"Urban sidewalk navigation presents significant challenges due to complex structural layouts, dynamic pedestrian behaviors, and long distances.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.16953","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Vail UCLA","organizationType":"academic-lab","sourceUrl":"https://vail-ucla.github.io/SidewalkBench/","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"general"},{"id":"bm_remembench_10890c1e","familyId":"bmf_c308a8d726bb","name":"ReMemBench","oneLine":"ReMemBench is a benchmark with eight diverse household manipulation tasks across four categories of short-term memory, designed to evaluate memory mechanisms in visuomotor policies.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.16178","pdf":"https://arxiv.org/pdf/2606.16178","project":"https://shahrutav.github.io/short-term-memory","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.16178"},"evidence":{"snippet":"To systematically evaluate memory in visuomotor control, we introduce ReMemBench -- a benchmark of eight diverse household manipulation tasks spanning four categories of short-term memory -- designed to foster general memory mechanisms rather than siloed, task-specific solutions.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16178"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ReMemBench is a benchmark with eight diverse household manipulation tasks across four categories of short-term memory, designed to evaluate memory mechanisms in visuomotor policies.","whyItMatters":"ReMemBench addresses the lack of systematic evaluation for short-term memory in visuomotor control, providing a standardized protocol to compare memory-augmented policies and guide development for long-horizon tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d050cfd6b7237b105bd4bc8089a78db5df54dacfd5a7061f8bf6e926c5336108"},"motivation":"Many robotic tasks require short-term memory, whether it's retrieving an object that's no longer visible or turning off an appliance after a set period.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.16178","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"PRISM team","organizationType":"academic-lab","sourceUrl":"https://shahrutav.github.io/short-term-memory","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_pal-bench_e5d27a42","familyId":"bmf_645decf62e6a","name":"PAL-Bench","oneLine":"PAL-Bench evaluates evidence-grounded profile reconstruction from longitudinal personal albums, scoring agents on owner facts, identities, and relations with a seven-metric protocol.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.16175","pdf":"https://arxiv.org/pdf/2606.16175","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.16175"},"evidence":{"snippet":"We introduce PAL-Bench, a controlled benchmark for evidence-grounded reconstruction under a public-record contract.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16175"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PAL-Bench evaluates evidence-grounded profile reconstruction from longitudinal personal albums, scoring agents on owner facts, identities, and relations with a seven-metric protocol.","whyItMatters":"Existing benchmarks test sub-problems of multimodal understanding, but PAL-Bench addresses the gap in album-scale reconstruction with social identity binding and evidence citation, offering a controlled public-record contract for evaluating perceptual entity resolution and multimodal integration.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d51d5404520fd53b33f34db135127f437a7ce01430b90c28ba184dc277b194bb"},"motivation":"Longitudinal personal albums are weak-schema multimodal databases: noisy perceptual records whose key facts require joins across faces, text, timestamps, locations, and repeated events.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.16175","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_p3b3_5a9c84e4","familyId":"bmf_35600c5fa04e","name":"P3B3","oneLine":"P3B3 is an expert-curated benchmark of conversational prompts for evaluating variety bias and controllability in LLMs across European and Brazilian Portuguese.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.16753","pdf":"https://arxiv.org/pdf/2606.16753","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.16753"},"evidence":{"snippet":"To address this gap, we introduce P3B3, an expert-curated language variety agnostic benchmark of conversational prompts, along with an evaluation framework for measuring variety bias and controllability.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16753"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"P3B3 is an expert-curated benchmark of conversational prompts for evaluating variety bias and controllability in LLMs across European and Brazilian Portuguese.","whyItMatters":"Evaluates regional variety bias in Portuguese LLMs, addressing underrepresentation and controllability gaps.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"59b0a456a8458ea20c319288e400dc669f34c77ec29b3087ec09ef53886424ea"},"motivation":"As Large Language Models (LLMs) become embedded in everyday communication, capturing regional linguistic variation is essential for reliable and equitable language use.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"MeLLM Workshop at ACL 2026","evidence":"Accepted at MeLLM Workshop at ACL 2026","evidenceUrl":"https://arxiv.org/abs/2606.16753","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"MeLLM Workshop at ACL 2026","reviewStatus":"accepted","decisionRaw":"Accepted at MeLLM Workshop at ACL 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.16753","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at MeLLM Workshop at ACL 2026","level":"author-claim"}]}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_memtrace_56029435","familyId":"bmf_cc4e1edaadda","name":"MemTrace","oneLine":"MemTrace evaluates long-term memory in LLM agents at the knowledge point level, probing memory age, question type, and evidence condition. Scoring is based on accuracy across these dimensions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.17328","pdf":"https://arxiv.org/pdf/2606.17328","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.17328"},"evidence":{"snippet":"We introduce MemTrace, a benchmark whose unit of measurement is the knowledge point: a single typed fact about the user, rather than an individual question.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.17328"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MemTrace evaluates long-term memory in LLM agents at the knowledge point level, probing memory age, question type, and evidence condition. Scoring is based on accuracy across these dimensions.","whyItMatters":"Aggregated accuracy misses important memory behaviors. MemTrace provides finer-grained metrics to reveal bottlenecks in evidence use, guiding improvements in memory systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"337defe4b68c90141b6c7e584e0aeee3beabd9ddf9ed8399eaa29a648260620d"},"motivation":"LLM agents increasingly maintain long-term memory of user facts across sessions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.17328","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_labosbench_5eb589a2","familyId":"bmf_89ea60d8b827","name":"LabOSBench","oneLine":"LabOSBench evaluates multimodal GUI agents on 96 subtasks across eight web-based scientific-instrument simulators, covering workflows from sample loading to result inspection. Agents operate via a browser, with execution-based evaluation on task completion.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.16802","pdf":"https://arxiv.org/pdf/2606.16802","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.16802"},"evidence":{"snippet":"To this end, we introduce LabOSBench, a challenging benchmark for multimodal GUI agents built on a suite of web-based scientific-instrument simulators.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16802"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LabOSBench evaluates multimodal GUI agents on 96 subtasks across eight web-based scientific-instrument simulators, covering workflows from sample loading to result inspection. Agents operate via a browser, with execution-based evaluation on task completion.","whyItMatters":"Existing computer-use benchmarks focus on software tasks, leaving a gap for scientific instrument control. LabOSBench provides a safe, reproducible, low-cost testbed to assess agents' feedback-driven and long-horizon capabilities in instrument operation, supporting practical adoption in laboratory automation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ddc0246d344550a2396fe637254122572a12cb621d64e6d5f4c3f52ffdb7aac5"},"motivation":"Current computer-use benchmarks primarily focus on software operation tasks in virtualized systems, whereas scientific instrumentation scenarios require coordinated control over complex interfaces, and feedback-driven parameter adjustment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.16802","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_hawkesnest_c85cfbd3","familyId":"bmf_8ce9fda1f146","name":"HawkesNest","oneLine":"HawkesNest is a synthetic benchmark for spatiotemporal point process models, providing controlled generators and complexity ladders across four axes: space-time entanglement, background heterogeneity, cross-type interaction, and domain topology. It includes simulation, export, and visualization tools.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.16863","pdf":"https://arxiv.org/pdf/2606.16863","project":null,"code":"https://github.com/YahyaAalaila/HawkesNest","data":null,"hfPaper":"https://huggingface.co/papers/2606.16863"},"evidence":{"snippet":"We introduce HawkesNest, a generator-aligned benchmark for controlled spatiotemporal pattern complexity built on a multivariate Hawkes backbone.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16863"},"ranking":{"90d":{"score":51,"rank":186,"coverage":0.55,"confidence":"Low"}},"description":"HawkesNest is a synthetic benchmark for spatiotemporal point process models, providing controlled generators and complexity ladders across four axes: space-time entanglement, background heterogeneity, cross-type interaction, and domain topology. It includes simulation, export, and visualization tools.","whyItMatters":"Real-world spatiotemporal event datasets obscure generative structure, making model failures hard to attribute. HawkesNest isolates complexity factors for diagnostic stress tests, enabling controlled evaluation of STPP models under known structural difficulty.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"105ff62c8a3491a08c68a64cf21d1e3522cc2d33183682e2806a5d0fc4231176"},"motivation":"Evaluation of spatiotemporal point process (STPP) models relies heavily on opaque real-world datasets, where latent generative structure is unknown and model failures are difficult to attribute.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.16863","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Yahya Aalaila et al.","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.16863","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_grace_1b12b544","familyId":"bmf_e010fd1ce1ac","name":"GRACE","oneLine":"GRACE evaluates step-level faithfulness of chain-of-thought reasoning in context-grounded tasks, with human annotations and a taxonomy of error categories.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.16151","pdf":"https://arxiv.org/pdf/2606.16151","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.16151"},"evidence":{"snippet":"We introduce GRACE, the first human-annotated step-level faithfulness benchmark with a data-driven error taxonomy for context-grounded textual reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16151"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GRACE evaluates step-level faithfulness of chain-of-thought reasoning in context-grounded tasks, with human annotations and a taxonomy of error categories.","whyItMatters":"Step-level faithfulness assessment addresses the gap where response-level metrics miss localized reasoning failures, providing granular feedback for improving model reliability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"81a05d96ef78b0f815e196a14a5027cbad43d39f4c52185407e45feb16e05b4d"},"motivation":"Many reasoning tasks require models to reason over input context, from document-grounded question answering to rule-based deduction.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.16151","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_geodisaster_9f255b61","familyId":"bmf_4fb86bec6f3a","name":"GeoDisaster","oneLine":"GeoDisaster is an operational geospatial disaster reasoning benchmark with 2,921 instances across 43 question types and five task families, integrating EO/GIS evidence and grounding answers in executable geospatial workflows.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.17246","pdf":"https://arxiv.org/pdf/2606.17246","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.17246"},"evidence":{"snippet":"We introduce GeoDisaster, an operational geospatial disaster reasoning benchmark with 2,921 verified instances across 43 question types and five task families: deforestation monitoring, multi-hazard analysis, building-damage assessment, flood-safe routing, and Sentinel-1 SAR flood monitoring.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.17246"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GeoDisaster is an operational geospatial disaster reasoning benchmark with 2,921 instances across 43 question types and five task families, integrating EO/GIS evidence and grounding answers in executable geospatial workflows.","whyItMatters":"GeoDisaster addresses the gap in evaluating tool-grounded spatial reasoning and structured decision-making for disaster response, offering a potential standard for assessing operational geo-intelligence in RS-VLMs and agentic systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d1405a2b503292a1256f72bd3557256f7ac39b107c88710272053284cc729e38"},"motivation":"Remote-sensing vision-language models (RS-VLMs) have advanced Earth-observation analysis toward visual interpretation and instruction-following, yet fall short of operational geo-intelligence, which demands tool-grounded spatial reasoning and structured, evidence-backed decisions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.17246","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_coffeebench_22526372","familyId":"bmf_43631014a768","name":"CoffeeBench","oneLine":"CoffeeBench evaluates LLM agents as a coffee roaster in a 90-day multi-agent economy with fixed reference agents, measuring cumulative net income through autonomous communication, negotiation, and transactions.","area":"Language & Knowledge","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Logistics"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.16613","pdf":"https://arxiv.org/pdf/2606.16613","project":null,"code":"https://github.com/SakanaAI/CoffeeBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.16613"},"evidence":{"snippet":"We introduce CoffeeBench, a benchmark for evaluating LLM agents in a long-horizon multi-agent economy composed of heterogeneous firms.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":10,"hfDailySubmittedAt":null,"githubStars":29,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16613"},"ranking":{"90d":{"score":70,"rank":75,"coverage":0.7,"confidence":"Medium"}},"description":"CoffeeBench evaluates LLM agents as a coffee roaster in a 90-day multi-agent economy with fixed reference agents, measuring cumulative net income through autonomous communication, negotiation, and transactions.","whyItMatters":"Existing benchmarks often focus on single agents in static environments, whereas CoffeeBench captures long-horizon multi-agent economic interactions. It provides a scoring contract based on net income, enabling comparison of agent economic decision-making and revealing failure modes like idle-drift.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b4cb6b5ad9df185d28eb61b7775d1b7d7587997c1f20c31ff01e45c2db9c5c89"},"motivation":"As LLM agents become capable of increasingly long-horizon tasks, evaluating their performance in economic systems is becoming increasingly important.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.16613","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Sakana AI","organizationType":"company-research-lab","sourceUrl":"https://github.com/SakanaAI/CoffeeBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_auau_d78e1117","familyId":"bmf_bcfe7d2ed921","name":"AuAu","oneLine":"AuAu combines psychometric instruments, vignettes, and realistic prompts to assess authoritarian tendencies in LLM responses, measuring sub-concepts like aggression, submission, and conventionalism.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.16127","pdf":"https://arxiv.org/pdf/2606.16127","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.16127"},"evidence":{"snippet":"We introduce AuAu, a comprehensive benchmark for assessing the risk of authoritarian tendencies in LLM responses.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16127"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"AuAu combines psychometric instruments, vignettes, and realistic prompts to assess authoritarian tendencies in LLM responses, measuring sub-concepts like aggression, submission, and conventionalism.","whyItMatters":"Provides a structured approach to auditing LLM authoritarian alignment, highlighting variations across models and the impact of system prompts on authoritarian output.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e73080a9d4422eb95d8fc245d6576a9d02d6f57e509aecebb04d3cf0516373f7"},"motivation":"The worldwide rise of authoritarianism and the growing role of Large Language Models (LLMs) in users' everyday lives raise the question of whether specific models exhibit or promote authoritarian attitudes.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.16127","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_atom-bench_82c6474b","familyId":"bmf_728bb9ed779a","name":"ATOM-Bench","oneLine":"ATOM-Bench evaluates atomic skills and compositional generalization in manipulation policies across 30 atomic tasks and 24 held-out compositional tasks, using paired single-arm and dual-arm robot tracks, with 3,000 human demonstrations and evaluation rollout data released.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.16826","pdf":"https://arxiv.org/pdf/2606.16826","project":"https://flageval-baai.github.io/AtomBenchPage","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.16826"},"evidence":{"snippet":"We introduce \\textbf{ATOM-Bench}, a real-world benchmark for evaluating both atomic skills and compositional generalization in manipulation policies.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16826"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ATOM-Bench evaluates atomic skills and compositional generalization in manipulation policies across 30 atomic tasks and 24 held-out compositional tasks, using paired single-arm and dual-arm robot tracks, with 3,000 human demonstrations and evaluation rollout data released.","whyItMatters":"ATOM-Bench provides a public diagnostic testbed for disentangling failures in motor execution, instruction grounding, and compositional reuse, which is crucial for advancing generalist manipulation policies beyond demonstrated tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a7d424463565a060cc14b7d35ed2016941916957d47ea8ff6f0f77684b046304"},"motivation":"Generalist manipulation policies are increasingly presented as foundation models for robotic control, but their real-world generalization remains difficult to diagnose.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.16826","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"BAAI","organizationType":"company-research-lab","sourceUrl":"https://flageval-baai.github.io/AtomBenchPage","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_agentfairbench_1a8330ea","familyId":"bmf_806b46eeb591","name":"AgentFairBench","oneLine":"AgentFairBench evaluates demographic disparity in the actions of LLM agents across hiring, lending, and medical triage. It uses synthetic, demographic-neutral profiles in counterfactual matched sets varying name-coded race/gender. Metrics include counterfactual flip rate, mean absolute score difference, action-rate disparity, and tool-invocation disparity, with bootstrap confidence intervals and FDR control.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-15","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.16723","pdf":"https://arxiv.org/pdf/2606.16723","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.16723"},"evidence":{"snippet":"We introduce AgentFairBench, a cheap, reproducible, multi-domain benchmark for demographic disparity in the actions of LLM agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.16723"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"AgentFairBench evaluates demographic disparity in the actions of LLM agents across hiring, lending, and medical triage. It uses synthetic, demographic-neutral profiles in counterfactual matched sets varying name-coded race/gender. Metrics include counterfactual flip rate, mean absolute score difference, action-rate disparity, and tool-invocation disparity, with bootstrap confidence intervals and FDR control.","whyItMatters":"Existing fairness evaluations grade answers, not actions. AgentFairBench addresses the gap by measuring disparity in consequential agent decisions. Its low cost and reproducible harness provide a practical path for screening models for action-level bias before deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"867d877722466981e8b6a05208f9e754896d1d37100e8809e931de311d2e2aa0"},"motivation":"Large language model (LLM) agents increasingly take actions (screening applicants, recommending credit, triaging patients), yet fairness for LLMs is still measured by grading answers.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"publication_reported","venue":"Under Review (2026)","evidence":"Under Review (2026)","evidenceUrl":"https://arxiv.org/abs/2606.16723","source":"arxiv-journal-reference","evidenceLevel":"strong-author-metadata","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publications":[{"venueName":"Under Review (2026)","publicationStatus":"published","evidence":[{"sourceType":"arxiv-journal-reference","sourceUrl":"https://arxiv.org/abs/2606.16723","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Under Review (2026)","level":"strong-author-metadata"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_urbanwell_81901465","familyId":"bmf_a905172c1364","name":"UrbanWell","oneLine":"UrbanWell provides a dataset and evaluation protocol for assessing spatio-temporal reasoning in multimodal large language models using satellite and street view imagery across 38 cities, covering environmental, accessibility, urban form, vitality, and subjective perception indicators with tasks in static prediction, forecasting, and trend classification.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15890","pdf":"https://arxiv.org/pdf/2606.15890","project":null,"code":"https://github.com/axin1301/UrbanWell-Benchmark","data":null,"hfPaper":"https://huggingface.co/papers/2606.15890"},"evidence":{"snippet":"We introduce UrbanWell, a large-scale benchmark designed to systematically evaluate the spatio-temporal reasoning capabilities of MLLMs for urban wellbeing analytics through joint modeling of satellite and street view imagery.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15890"},"ranking":{"90d":{"score":51,"rank":190,"coverage":0.7,"confidence":"Medium"}},"description":"UrbanWell provides a dataset and evaluation protocol for assessing spatio-temporal reasoning in multimodal large language models using satellite and street view imagery across 38 cities, covering environmental, accessibility, urban form, vitality, and subjective perception indicators with tasks in static prediction, forecasting, and trend classification.","whyItMatters":"UrbanWell addresses the lack of standardized benchmarks for multimodal urban wellbeing analytics, enabling consistent comparison of MLLMs on tasks requiring joint spatial and temporal understanding. This supports progress in urban intelligence applications such as planning and policy evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"38d41b8df9c86e25edea6a2ba6336fb02203463e7bf4f1b501b2881ba9dd8ec0"},"motivation":"Understanding urban wellbeing from multimodal data requires integrating heterogeneous spatial and temporal signals, posing significant challenges for current multimodal large language models (MLLMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"KDD Datasets and Benchmarks Track 2026","evidence":"accepted by KDD Datasets and Benchmarks Track 2026","evidenceUrl":"https://arxiv.org/abs/2606.15890","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"KDD Datasets and Benchmarks Track 2026","reviewStatus":"accepted","decisionRaw":"accepted by KDD Datasets and Benchmarks Track 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.15890","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"accepted by KDD Datasets and Benchmarks Track 2026","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_tickingcollabbench_3fac6f85","familyId":"bmf_b34cf6b2cdf3","name":"TickingCollabBench","oneLine":"Evaluates multi-agent collaboration in Minecraft-based tasks with time-sensitive complementary collaboration, requiring agent heterogeneity, mandatory collaboration, and dynamic environments. The framework generates diverse tasks via YAML specifications and filters invalid configurations.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15684","pdf":"https://arxiv.org/pdf/2606.15684","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.15684"},"evidence":{"snippet":"We present TickingCollabBench, a Minecraft-based multi-agent benchmark for a novel class of time-sensitive complementary collaboration tasks.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15684"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates multi-agent collaboration in Minecraft-based tasks with time-sensitive complementary collaboration, requiring agent heterogeneity, mandatory collaboration, and dynamic environments. The framework generates diverse tasks via YAML specifications and filters invalid configurations.","whyItMatters":"Bridges the gap between static multi-agent benchmarks and real-world scenarios needing real-time coordination under partial observability. Provides a feasibility-aware pipeline for reproducible task generation and evaluation of LLM-based agents in dynamic settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"4f0a9c0e7a5b720b18a5aec27c86c5305ad5aa7ddbbf168a822d07409896ddd7"},"motivation":"We present TickingCollabBench, a Minecraft-based multi-agent benchmark for a novel class of time-sensitive complementary collaboration tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15684","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TickingCollabBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.15684","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_retailbench_1e87a5e2","familyId":"bmf_9a9797d125ac","name":"RetailBench","oneLine":"RetailBench is a simulation benchmark evaluating tool-using LLM agents in single-store supermarket operations over a 180-day horizon. It covers pricing, replenishment, supplier selection, assortment, inventory aging, customer feedback, external events, and cash-flow constraints, with a privileged oracle policy for comparison.","area":"Language & Knowledge","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Logistics"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15862","pdf":"https://arxiv.org/pdf/2606.15862","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.15862"},"evidence":{"snippet":"We introduce RetailBench, a data-grounded simulation benchmark for evaluating tool-using LLM agents in single-store supermarket operation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15862"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RetailBench is a simulation benchmark evaluating tool-using LLM agents in single-store supermarket operations over a 180-day horizon. It covers pricing, replenishment, supplier selection, assortment, inventory aging, customer feedback, external events, and cash-flow constraints, with a privileged oracle policy for comparison.","whyItMatters":"RetailBench addresses the gap in evaluating LLM agents on long-horizon, economically grounded decision-making, where short-horizon tasks dominate existing benchmarks. It provides a controlled testbed for assessing coherent decision-making and reliability in dynamic environments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4cb9393ccb61d88fc5a48fcfeeac409e38663318a036a3fe9a0c2277a5b06b7d"},"motivation":"Large language model (LLM) agents have made rapid progress on short-horizon, well-scoped tasks, yet their ability to sustain coherent decisions in dynamic long-horizon environments remains uncertain.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15862","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_omnitraffic_61df9afe","familyId":"bmf_8c1a75de50ea","name":"OmniTraffic","oneLine":"OmniTraffic is a controllable generation pipeline and benchmark for spatio-temporal traffic reasoning. It provides 8M VQA samples and a 3K human-verified test set across 12 reconstructed 3D intersections, with a three-level task hierarchy covering scene perception, multi-view and temporal reasoning, and decision support.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15749","pdf":"https://arxiv.org/pdf/2606.15749","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.15749"},"evidence":{"snippet":"We introduce OmniTraffic, a controllable generation pipeline and benchmark for spatio-temporal traffic reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15749"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OmniTraffic is a controllable generation pipeline and benchmark for spatio-temporal traffic reasoning. It provides 8M VQA samples and a 3K human-verified test set across 12 reconstructed 3D intersections, with a three-level task hierarchy covering scene perception, multi-view and temporal reasoning, and decision support.","whyItMatters":"OmniTraffic fills the gap in evaluating structure-aware traffic reasoning under controlled conditions, where existing traffic benchmarks focus on passive recognition. The extensible pipeline allows configurable scenarios for reproducible evaluation and simulation-generated supervision for model fine-tuning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"101a4b0d802cbf08a3ae456d3d0b48bd4f9708849d172bcec6ffb102bc7112ae"},"motivation":"Traffic scene understanding requires models to reason beyond object recognition, including lane topology, multi-view geometry, temporal evolution, and signal-phase semantics.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15749","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_finbalance_0aab2d5b","familyId":"bmf_5dac25fe40d5","name":"FinBalance","oneLine":"FinBalance is a multi-document accounting reconciliation benchmark built from 710 source-document bundles across eight industries, three period types, and five difficulty levels. It evaluates models on producing journal entries, balance sheets, and inconsistency labels, with a deterministic ledger for ground truth.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-06-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15949","pdf":"https://arxiv.org/pdf/2606.15949","project":null,"code":"https://github.com/Devansh1105/finbalance","data":null,"hfPaper":"https://huggingface.co/papers/2606.15949"},"evidence":{"snippet":"We introduce FinBalance, a multi-document accounting reconciliation benchmark built from source-document bundles across eight industries, three period types, and five difficulty levels.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15949"},"ranking":{"90d":{"score":29,"rank":375,"coverage":0.55,"confidence":"Low"}},"description":"FinBalance is a multi-document accounting reconciliation benchmark built from 710 source-document bundles across eight industries, three period types, and five difficulty levels. It evaluates models on producing journal entries, balance sheets, and inconsistency labels, with a deterministic ledger for ground truth.","whyItMatters":"FinBalance addresses the gap in evaluating accounting reasoning from source documents rather than prepared financial statements. It provides a reproducible and auditable benchmark for document-grounded financial reasoning, with expert validation of the design.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"171a94eddc5ceeb2b414d1545b860dda7e33b91878e40f643dd7a686f4955d38"},"motivation":"Existing financial-NLP benchmarks mostly evaluate prepared artifacts such as filings, tables, or extracted values.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15949","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FinBalance Team","organizationType":"academic-lab","sourceUrl":"https://github.com/Devansh1105/finbalance","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_eibench_cb3bf853","familyId":"bmf_1505489eac0d","name":"EIBench","oneLine":"EIBench is a simulator-based benchmark for interactive emotion management, containing 2,222 scenarios across a 2x2 taxonomy of Support, Defense, Repair, and Charm. It evaluates LLM agents on multi-turn dialogue where a user simulator updates emotion-relation states and provides anchor-based scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15532","pdf":"https://arxiv.org/pdf/2606.15532","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.15532"},"evidence":{"snippet":"We introduce EIBench, a simulator-based benchmark for interactive emotion management.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15532"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"EIBench is a simulator-based benchmark for interactive emotion management, containing 2,222 scenarios across a 2x2 taxonomy of Support, Defense, Repair, and Charm. It evaluates LLM agents on multi-turn dialogue where a user simulator updates emotion-relation states and provides anchor-based scoring.","whyItMatters":"EIBench fills the gap in evaluating emotional intelligence beyond static understanding, focusing on interactive emotion management over multiple turns. Its simulator provides both outcome and dense turn-level feedback, enabling training and evaluation in a unified environment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f83dd1f01dbe8123af75524a9e92e5e33cf36cd022a2ea59799fa35048fcb238"},"motivation":"Emotional intelligence (EI) in Large Language Models (LLMs) is often evaluated through static understanding tasks or single-response dialogue generation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15532","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_ehrnote-chatqa_748f904b","familyId":"bmf_6bcd9830c15b","name":"EHRNote-ChatQA","oneLine":"EHRNote-ChatQA is a benchmark for evidence-grounded multi-turn clinical QA over longitudinal discharge summaries. Built from MIMIC-IV, it includes 967 patient-level samples and 16,072 expert-verified QA pairs across eight clinical categories, with evidence-grounding QA pairs.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15735","pdf":"https://arxiv.org/pdf/2606.15735","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.15735"},"evidence":{"snippet":"We introduce EHRNote-ChatQA, the first benchmark for evidence-grounded multi-turn clinical question answering over patients' multiple discharge summaries.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15735"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EHRNote-ChatQA is a benchmark for evidence-grounded multi-turn clinical QA over longitudinal discharge summaries. Built from MIMIC-IV, it includes 967 patient-level samples and 16,072 expert-verified QA pairs across eight clinical categories, with evidence-grounding QA pairs.","whyItMatters":"EHRNote-ChatQA addresses the gap in evaluating clinical QA systems on multi-turn, evidence-grounded reasoning over multiple documents, which reflects real clinical review workflows. It provides a rigorous benchmark for assessing evidence grounding and the compounding of errors over turns.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"30890303f5a28845d1ffb75f3a6909cb92ebca9063463fb54f4261ca16723aea"},"motivation":"Discharge summaries are crucial clinical documents containing the context of a patient's overall hospital stay, and are routinely reviewed by medical experts for patient readmission, ongoing care, and diagnostic decision-making.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15735","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_cogcanvas_d3de924a","familyId":"bmf_5fd28976da11","name":"CogCanvas","oneLine":"CogCanvas is a benchmark for multi-subject reference-based image generation, with 1,952 curated reference images, 1,361 compositional prompts, and a unified six-axis evaluation protocol. It includes tasks for reference-based generation, text-to-image composition, and reference retrieval, with metrics BG-Sim and Attr-VQA.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-14","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15867","pdf":"https://arxiv.org/pdf/2606.15867","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.15867"},"evidence":{"snippet":"We introduce CogCanvas, a benchmark of 1,952 curated reference images spanning 100 celebrity identities, 115 distinctive objects and fashion items, and 29 real-world background scenes including landmarks, from which we construct 1,361 compositional prompts covering 2-5 person group sizes.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15867"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CogCanvas is a benchmark for multi-subject reference-based image generation, with 1,952 curated reference images, 1,361 compositional prompts, and a unified six-axis evaluation protocol. It includes tasks for reference-based generation, text-to-image composition, and reference retrieval, with metrics BG-Sim and Attr-VQA.","whyItMatters":"CogCanvas addresses the gap in jointly evaluating multi-identity, object binding, background grounding, and spatial plausibility in image generation. It provides a comprehensive benchmark for assessing composition capabilities across group sizes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"399d38ad19fb4377b9470179b792d6a403db3f0c94795b3db9317e1669b49650"},"motivation":"Multi-subject reference-based image generation requires jointly preserving multiple human identities, binding per-person objects and fashion items, and respecting a specified background scene, a regime where current diffusion models remain brittle.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15867","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_toolmenubench_fce3386d","familyId":"bmf_e8024a0dd932","name":"ToolMenuBench","oneLine":"ToolMenuBench is a benchmark for evaluating tool-menu filtering strategies in multi-step LLM agents. It varies tool-menu size, distractor type, state-dependent structure, and risk exposure, and reports filter-level and downstream metrics such as task success, tool calls, and token usage.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15508","pdf":"https://arxiv.org/pdf/2606.15508","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.15508"},"evidence":{"snippet":"We introduce ToolMenuBench, a benchmark for evaluating tool-menu construction in multi-step LLM agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15508"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ToolMenuBench is a benchmark for evaluating tool-menu filtering strategies in multi-step LLM agents. It varies tool-menu size, distractor type, state-dependent structure, and risk exposure, and reports filter-level and downstream metrics such as task success, tool calls, and token usage.","whyItMatters":"ToolMenuBench addresses the gap in evaluating how tool-menu construction affects reliability, efficiency, and risk in tool-augmented agents. It provides a reusable framework for studying the agent-interface problem with controlled settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7b4704731947dee77ddf9a0cbe3f560a97d6c8155d3759292525b60bfe91cbb9"},"motivation":"Tool-augmented large language model agents increasingly operate over large tool libraries, but existing evaluations often focus on whether a model can call a tool correctly rather than how the visible tool menu shapes reliability, efficiency, and safety-relevant risk exposure.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15508","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_scr-bench_984f0fe8","familyId":"bmf_ae50c3042f6a","name":"SCR-Bench","oneLine":"SCR-Bench is a benchmark for evaluating security risks in composed LLM agent skill workflows. It includes three sub-benchmarks (SCR-CapFlow, SCR-TrustLift, SCR-AuthBlur) that measure attack success rates, trust transfer, and authorization confusion in sandboxed environments.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15242","pdf":"https://arxiv.org/pdf/2606.15242","project":null,"code":"https://github.com/saint-viperx/SCR_Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.15242"},"evidence":{"snippet":"We introduce SCR-Bench to evaluate this risk in controlled, sandboxed skill environments.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":14,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15242"},"ranking":{"90d":{"score":57,"rank":156,"coverage":0.7,"confidence":"Medium"}},"description":"SCR-Bench is a benchmark for evaluating security risks in composed LLM agent skill workflows. It includes three sub-benchmarks (SCR-CapFlow, SCR-TrustLift, SCR-AuthBlur) that measure attack success rates, trust transfer, and authorization confusion in sandboxed environments.","whyItMatters":"SCR-Bench addresses the gap in evaluating agent skill security at the path level, where skills benign in isolation become harmful in composition. It provides a controlled, sandboxed environment for assessing composition-induced risks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"aa29672772df59586e700ba9db9bc6f2001ea20ee085d0daea2c6c162a3a8f47"},"motivation":"Skills are becoming the capability layer through which LLM agents turn plans into actions, but their use introduces security risks such as data leakage, unauthorized operations, and tool misuse.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15242","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SCR-Bench Team","organizationType":"academic-lab","sourceUrl":"https://github.com/saint-viperx/SCR_Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_pacute_04643d6a","familyId":"bmf_e2d2cfe524a8","name":"PACUTE","oneLine":"PACUTE is a diagnostic benchmark of 4,600 tasks evaluating morphological understanding in Filipino, covering six compositional levels from morpheme decomposition to syllabification.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15144","pdf":"https://arxiv.org/pdf/2606.15144","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.15144"},"evidence":{"snippet":"We introduce PACUTE, a diagnostic benchmark of 4,600 tasks designed to evaluate morphological understanding in Filipino, a language characterized by productive infixation, reduplication, and diacritic-driven lexical distinctions that are typically absent from written text.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15144"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PACUTE is a diagnostic benchmark of 4,600 tasks evaluating morphological understanding in Filipino, covering six compositional levels from morpheme decomposition to syllabification.","whyItMatters":"Standard tokenizers obscure character-level and morphological structure, particularly for languages with non-concatenative morphology. This benchmark localizes where morphological understanding breaks down, distinguishing character access from productive composition, which informs tokenizer and model design for low-resource languages.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d402990649c1042705b9b5ec40901b209d1aac7afe5aa902bc0e4af10aee0853"},"motivation":"Large language models (LLMs) process text as sequences of subword tokens, which can obscure the character-level and morphological structure that underlies word formation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15144","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_osguard_1e29e1e9","familyId":"bmf_e3e2aee3f6ec","name":"OSGuard","oneLine":"OSGuard is a dual-granularity benchmark suite for evaluating safety in computer-use agents, with an action-level benchmark for local guardrail decisions and a risk-augmented execution suite for end-to-end evaluation.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15034","pdf":"https://arxiv.org/pdf/2606.15034","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.15034"},"evidence":{"snippet":"We introduce OSGuard, a dual-granularity benchmark suite for evaluating safety in computer-use agents under benign, unchanged user instructions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15034"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OSGuard is a dual-granularity benchmark suite for evaluating safety in computer-use agents, with an action-level benchmark for local guardrail decisions and a risk-augmented execution suite for end-to-end evaluation.","whyItMatters":"Task success alone misses unsafe shortcuts in computer-use agents. OSGuard's dual-granularity design distinguishes local recognition of unsafe actions from full-task safety, providing a more precise diagnosis of guardrail capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2e49568fa636d1d7c39023ab6f3edee2ec0ee1461aebb46c5b08980ba8e3c718"},"motivation":"Computer-use agents are increasingly evaluated by whether they complete realistic desktop and web tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15034","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_irts-toolbench_94993a2c","familyId":"bmf_cce157d7294f","name":"IRTS-ToolBench","oneLine":"IRTS-ToolBench is a benchmark of 1,700 questions across 10 task types and 13 domains for evaluating irregular univariate time-series question answering, with standardized inputs and a reproducible evaluation protocol.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15107","pdf":"https://arxiv.org/pdf/2606.15107","project":null,"code":"https://github.com/SanhornC/IRTS-ToolBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.15107"},"evidence":{"snippet":"To bridge this gap, we introduce IRTS-ToolBench, a benchmark of 1,700 questions spanning 10 task types across 13 domains.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15107"},"ranking":{"90d":{"score":39,"rank":295,"coverage":0.55,"confidence":"Low"}},"description":"IRTS-ToolBench is a benchmark of 1,700 questions across 10 task types and 13 domains for evaluating irregular univariate time-series question answering, with standardized inputs and a reproducible evaluation protocol.","whyItMatters":"Existing TSQA benchmarks assume regular sampling, leaving a gap for real-world irregular data. This benchmark provides standardized evaluation for LLMs and AI agents on irregular time series, with golden tool sets for tool-selection analysis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7f0189ab569b5b0bf93070aa3fde64a64e744f7b6446d8edd1949923a10a6465"},"motivation":"Time series data in real-world deployments is overwhelmingly irregular.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15107","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_envship_5107b438","familyId":"bmf_467188c4c8c6","name":"EnvShip","oneLine":"EnvShip is a unified multi-region framework for context-aware and cross-region vessel trajectory forecasting, with standardized tracks and evaluation protocols using public AIS data.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-13","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.15240","pdf":"https://arxiv.org/pdf/2606.15240","project":null,"code":null,"data":"https://huggingface.co/datasets/mark000071/envship_v2_datasets","hfPaper":"https://huggingface.co/papers/2606.15240"},"evidence":{"snippet":"EnvShip provides a common and reproducible testbed for vessel trajectory forecasting.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":356,"hfDatasetLikes":2},"source":{"type":"arxiv","id":"2606.15240"},"ranking":{"90d":{"score":57,"rank":149,"coverage":0.3,"confidence":"Low","datasetDownloadRank":14,"datasetRankPopulation":51}},"description":"EnvShip is a unified multi-region framework for context-aware and cross-region vessel trajectory forecasting, with standardized tracks and evaluation protocols using public AIS data.","whyItMatters":"Existing studies use incompatible preprocessing and evaluation settings, making results difficult to compare. EnvShip provides a common, reproducible testbed for vessel trajectory forecasting, enabling fair comparison across methods and regions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2eb2534910e8fc56505bdc8b26b89179a1df9f475981ff6e54c6a55d50403a10"},"motivation":"Accurate vessel trajectory forecasting is essential for maritime situational awareness, navigation safety, traffic management, and autonomous navigation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15240","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_comet-bench_355dabd3","familyId":"bmf_2bde53728ad5","name":"CoMET-Bench","oneLine":"CoMET-Bench is a benchmark for conditional multi-event temporal grounding in long-form video, with 2,789 queries over 600 videos and a unified evaluation protocol including counting, grounding, and negative-query recognition.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15320","pdf":"https://arxiv.org/pdf/2606.15320","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.15320"},"evidence":{"snippet":"We introduce CoMET-Bench for Conditional Multi-Event Temporal Grounding in long-form video, comprising 2789 queries over 600 videos averaging 33.8 minutes across five real-world domains, with each query composed from 4 temporal conditions, 3 spatial conditions, and a dedicated negative-query subset.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15320"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CoMET-Bench is a benchmark for conditional multi-event temporal grounding in long-form video, with 2,789 queries over 600 videos and a unified evaluation protocol including counting, grounding, and negative-query recognition.","whyItMatters":"Real-world video grounding requires localizing every event satisfying compositional conditions, which existing benchmarks do not jointly handle. This benchmark introduces Rejection-F1 to prevent trivial gaming and exposes gaps in current methods.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"58c5389c132002ab78491d9e7721fe06ff02127749af1a073657480c0b943e1d"},"motivation":"Multimodal large language models have made rapid progress in video temporal grounding, yet real-world applications routinely require localizing every event that satisfies compositional temporal and spatial conditions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15320","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_coda-bench_a3ad2891","familyId":"bmf_7e9ed12d69ca","name":"CODA-BENCH","oneLine":"CODA-Bench is a benchmark for evaluating AI agents on data-intensive analytical tasks in a Linux sandbox with 1,009 tasks across 31 communities, requiring data discovery, code generation, and correct answers.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15300","pdf":"https://arxiv.org/pdf/2606.15300","project":"https://coda-bench.github.io/","code":"https://github.com/ruc-datalab/CoDA-Bench","data":"https://huggingface.co/datasets/RUC-DataLab/CoDA-Bench","hfPaper":"https://huggingface.co/papers/2606.15300"},"evidence":{"snippet":"In this paper, we bridge this gap by introducing CODA-BENCH, the first benchmark to jointly evaluate code and data intelligence in a data-intensive environment.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":13,"hfDailySubmittedAt":"2026-06-16T00:00:00.000Z","githubStars":43,"githubScope":"benchmark_repo","hfDatasetDownloads":403,"hfDatasetLikes":2},"source":{"type":"arxiv","id":"2606.15300"},"ranking":{"90d":{"score":81,"rank":18,"coverage":1.0,"confidence":"High","datasetDownloadRank":13,"datasetRankPopulation":51}},"description":"CODA-Bench is a benchmark for evaluating AI agents on data-intensive analytical tasks in a Linux sandbox with 1,009 tasks across 31 communities, requiring data discovery, code generation, and correct answers.","whyItMatters":"Existing benchmarks evaluate code or data capabilities in isolation. CODA-Bench jointly evaluates both, reflecting real development scenarios with complex file systems and large-scale data, revealing gaps in agentic data intelligence.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bc53bacf180f2fd3e57bb930c67527a8ab71526a260aaa2104150d7856741a38"},"motivation":"Advanced agents are increasingly demonstrating the potential to operate as autonomous engineers, creating a growing demand for evaluation benchmarks that capture the complexity of real-world development.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026","evidence":"Accepted at ICML 2026. 37 pages, 11 figures. Project page: https://coda-bench.github.io/ Code: https://github.com/ruc-datalab/CoDA-Bench Data: https://huggingface.co/datasets/RUC-DataLab/CoDA-Bench","evidenceUrl":"https://arxiv.org/abs/2606.15300","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026","reviewStatus":"accepted","decisionRaw":"Accepted at ICML 2026. 37 pages, 11 figures. Project page: https://coda-bench.github.io/ Code: https://github.com/ruc-datalab/CoDA-Bench Data: https://huggingface.co/datasets/RUC-DataLab/CoDA-Bench","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.15300","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at ICML 2026. 37 pages, 11 figures. Project page: https://coda-bench.github.io/ Code: https://github.com/ruc-datalab/CoDA-Bench Data: https://huggingface.co/datasets/RUC-DataLab/CoDA-Bench","level":"author-claim"}]}],"publishers":[{"name":"RUC-DataLab","organizationType":"academic-lab","sourceUrl":"https://github.com/ruc-datalab/CoDA-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_amchibias_ca7c8175","familyId":"bmf_6474c5a141a3","name":"AmchiBias","oneLine":"AmchiBias is a benchmark for measuring socio-cultural stereotypical bias for Goan identity groups, with 313 minimal pairs across eight sociodemographic dimensions in English and Devanagari Konkani.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-13","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.15191","pdf":"https://arxiv.org/pdf/2606.15191","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.15191"},"evidence":{"snippet":"We present AmchiBias, the first benchmark for measuring socio-cultural stereotypical bias for the Indian state of Goa with its unique historically multicultural setting.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.15191"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AmchiBias is a benchmark for measuring socio-cultural stereotypical bias for Goan identity groups, with 313 minimal pairs across eight sociodemographic dimensions in English and Devanagari Konkani.","whyItMatters":"Stereotypical bias is often considered only at the national level. This benchmark addresses hyperlocal subnational identities in a low-resource language, highlighting gaps in multilingual model evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fad2528b95569de9277ecb132960a167370706b96a97010238b3505dfc591884"},"motivation":"Socio-cultural stereotypical bias is an important consideration in the development and deployment of NLP systems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.15191","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_streammembench_6345b534","familyId":"bmf_9ab5cd1646d9","name":"StreamMemBench","oneLine":"StreamMemBench is a streaming benchmark for evaluating agent memory with two-step task sequences around evidence anchors from EgoLife egocentric streams, with four diagnostic metrics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.14571","pdf":"https://arxiv.org/pdf/2606.14571","project":null,"code":"https://github.com/landian60/StreamMemBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.14571"},"evidence":{"snippet":"We introduce StreamMemBench, a streaming benchmark that constructs a two-step task sequence around each evidence anchor from EgoLife egocentric streams.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":24,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.14571"},"ranking":{"90d":{"score":61,"rank":131,"coverage":0.7,"confidence":"Medium"}},"description":"StreamMemBench is a streaming benchmark for evaluating agent memory with two-step task sequences around evidence anchors from EgoLife egocentric streams, with four diagnostic metrics.","whyItMatters":"Existing memory benchmarks test recall or task improvement in isolation. StreamMemBench evaluates the trajectory from streaming observations to future-oriented assistance, diagnosing whether memory systems use evidence and incorporate feedback.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1e9604ff519227981ed24535395a4b69998f542eff33a3c1122d816caba19716"},"motivation":"A central role of personal-agent memory is to turn stored information and prior interactions into future-oriented assistance.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.14571","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_simmer_2d9d2c96","familyId":"bmf_365fcc9a87e9","name":"SIMMER","oneLine":"SIMMER evaluates latent failures in LLM-generated plans for kitchen-domain tasks using a curated symbolic world model with 77 actions and 262 objects, scoring error-free plans and latent hazard detection.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning"],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.14574","pdf":"https://arxiv.org/pdf/2606.14574","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.14574"},"evidence":{"snippet":"To address this gap, we introduce SIMMER, a benchmark for evaluating latent failures in LLM planning through a human-curated symbolic world model grounded in the kitchen domain.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.14574"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SIMMER evaluates latent failures in LLM-generated plans for kitchen-domain tasks using a curated symbolic world model with 77 actions and 262 objects, scoring error-free plans and latent hazard detection.","whyItMatters":"Existing plan benchmarks miss failures that don't immediately halt execution but compromise goals; SIMMER provides metrics for irreversible latent failures, important for safe deployment of LLM planners.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"354641bae5f4dfcc81b7856be20ae40c7a1448dfa8fe7042ce46c55500d3bc10"},"motivation":"Large language models (LLMs) are increasingly deployed as planners for autonomous agents in household environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"COLM 2026","evidence":"Accepted at COLM 2026","evidenceUrl":"https://arxiv.org/abs/2606.14574","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"COLM 2026","reviewStatus":"accepted","decisionRaw":"Accepted at COLM 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.14574","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at COLM 2026","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_reactsim-bench_5401cd81","familyId":"bmf_a6b25ed93a48","name":"ReactSim-Bench","oneLine":"ReactSim-Bench evaluates reactive capability of behavior world model simulators in autonomous driving by decoupling agent and AV control, using collision, map, and kinematic metrics on 2,636 scenarios.","area":"Robotics & Embodied AI","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Automotive"],"capabilities":[],"topics":["cs.RO"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-12","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.14058","pdf":"https://arxiv.org/pdf/2606.14058","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.14058"},"evidence":{"snippet":"In this work, we introduce ReactSim-Bench for evaluating the reactive capability of behavior world model simulation in autonomous driving.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.14058"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ReactSim-Bench evaluates reactive capability of behavior world model simulators in autonomous driving by decoupling agent and AV control, using collision, map, and kinematic metrics on 2,636 scenarios.","whyItMatters":"Existing sim benchmarks don't directly measure whether simulated agents respond feasibly to novel AV behaviors; this benchmark fills that gap for safe simulation-based testing.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9dd59cb24a0c58188de4590d7f9edced5b30bda4b5d859f8ff45e157eb2da664"},"motivation":"Reactive capability is a key property of data-driven behavior world model simulators for autonomous driving simulation systems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.14058","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_phoneharness_125a2bf7","familyId":"bmf_a6d0de088cce","name":"PhoneHarness","oneLine":"PhoneHarness Bench evaluates phone-use agents on verifiable mobile workflows with mixed GUI, CLI, and tool actions, scored by observable side effects from auditable execution traces.","area":"Agents & Tool Use","applicationDomains":["Consumer & Productivity"],"primaryDomain":"Consumer & Productivity","industrySectors":["Consumer Technology"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.14832","pdf":"https://arxiv.org/pdf/2606.14832","project":"https://phoneharness.github.io/","code":"https://github.com/PhoneHarness/PhoneHarness","data":null,"hfPaper":"https://huggingface.co/papers/2606.14832"},"evidence":{"snippet":"We introduce PhoneHarness, a mixed-action benchmark and execution harness for studying phone-use agents on verifiable mobile workflows.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":12,"hfDailySubmittedAt":"2026-06-16T00:00:00.000Z","githubStars":47,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.14832"},"ranking":{"90d":{"score":74,"rank":53,"coverage":0.7,"confidence":"Medium"}},"description":"PhoneHarness Bench evaluates phone-use agents on verifiable mobile workflows with mixed GUI, CLI, and tool actions, scored by observable side effects from auditable execution traces.","whyItMatters":"Mobile agent evaluation often ignores non-GUI actions and side effects; this benchmark measures complete task completion in real device environments, filling a gap for reliable phone automation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"88de4646e887ccf34184dcc526f7bf49218637039c4f01fe9b09cb182befd1cc"},"motivation":"Phone agents are increasingly expected to complete real mobile workflows rather than merely predict the next screen action.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.14832","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_losona_d944301c","familyId":"bmf_d46a125f03ce","name":"LoSoNA","oneLine":"LoSoNA evaluates LLM agents' ability to infer and adapt to local social norms in group chats, using curated transcripts and an elicitor turn to score norm-conforming responses.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.14600","pdf":"https://arxiv.org/pdf/2606.14600","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.14600"},"evidence":{"snippet":"We introduce LoSoNA, a benchmark for local social norm adaptation in multi-party chat.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":7,"hfDailySubmittedAt":"2026-06-15T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.14600"},"ranking":{"90d":{"score":53,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"LoSoNA evaluates LLM agents' ability to infer and adapt to local social norms in group chats, using curated transcripts and an elicitor turn to score norm-conforming responses.","whyItMatters":"LLMs in social contexts must recognize unstated norms; this benchmark probes a capability that existing social benchmarks overlook, important for believable agent behavior.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cb381a3598a64e0209c366ffd430bcf8d7a451b7b81f1ed3fa84735f3db7a0d6"},"motivation":"Online group chats are social spaces with local conversational norms that are rarely stated explicitly.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.14600","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_industrybench-mipu_6704f5ae","familyId":"bmf_6260f7570604","name":"IndustryBench-MIPU","oneLine":"IndustryBench-MIPU evaluates MLLMs on extracting structured attribute-value pairs from multi-image industrial product data, covering text recognition, visual reasoning, and cross-image integration.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.14383","pdf":"https://arxiv.org/pdf/2606.14383","project":null,"code":"https://github.com/alibaba-multimodal-industrial-ai/IndustryBench-MIPU","data":null,"hfPaper":"https://huggingface.co/papers/2606.14383"},"evidence":{"snippet":"To fill this gap, we introduce IndustryBench-MIPU, the first large-scale benchmark for multi-image industrial product understanding, built around structured attribute extraction -- recovering property-value pairs from product images.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":4,"hfDailySubmittedAt":"2026-06-18T00:00:00.000Z","githubStars":11,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.14383"},"ranking":{"90d":{"score":61,"rank":124,"coverage":0.7,"confidence":"Medium"}},"description":"IndustryBench-MIPU evaluates MLLMs on extracting structured attribute-value pairs from multi-image industrial product data, covering text recognition, visual reasoning, and cross-image integration.","whyItMatters":"Industrial product specifications are scattered across images, and current MLLMs show a recall gap; this benchmark quantifies multi-image understanding limits for procurement and safety.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f62a1ce1637e2cb638f2f123a1f25a2591353c98cd163fea25419178a97024a7"},"motivation":"Industrial products such as valves and circuit breakers are defined by dense technical specifications that govern procurement, compatibility, and safety across supply chains.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.14383","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Alibaba","organizationType":"company-research-lab","sourceUrl":"https://github.com/alibaba-multimodal-industrial-ai/IndustryBench-MIPU","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_gauntletbench_be5725f6","familyId":"bmf_44f425325bfc","name":"GauntletBench","oneLine":"GauntletBench evaluates agent generalisation across five professional web applications with 100 vision-intensive tasks, probing temporal perception, graphical understanding, and 3D reasoning via automated objective scoring.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Geometric reasoning"],"topics":["Agents","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.14397","pdf":"https://arxiv.org/pdf/2606.14397","project":null,"code":"https://github.com/gauntlet-benchmark/evaluation-harness","data":null,"hfPaper":"https://huggingface.co/papers/2606.14397"},"evidence":{"snippet":"To this end, we introduce GauntletBench, a web-based benchmark for evaluating agent generalisation in challenging scenarios, focusing on three underexplored capabilities (temporal perception, graphical understanding, and 3D reasoning), across five less-covered professional applications (Video Editor, Workflow Builder, 3D Modeller, Flight Analyser, and Circuit Designer), each with 20 vision-intensive tasks (100 in total).","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":19,"hfDailySubmittedAt":"2026-06-26T00:00:00.000Z","githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.14397"},"ranking":{"90d":{"score":44,"rank":237,"coverage":0.7,"confidence":"Medium"}},"description":"GauntletBench evaluates agent generalisation across five professional web applications with 100 vision-intensive tasks, probing temporal perception, graphical understanding, and 3D reasoning via automated objective scoring.","whyItMatters":"Existing agent benchmarks saturate and overlook harder capabilities; this benchmark reveals significant gaps in frontier agents, guiding development toward more robust real-world systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9993efbd5567a9dad8a4342b89529f7361b30868cc125412d1cfa395673fe155"},"motivation":"As agentic systems continue to evolve and are widely deployed in real-world scenarios, there is a growing demand to faithfully evaluate their capabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.14397","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_gama-bench_3cec2065","familyId":"bmf_e443c7d96cbb","name":"GAMA-Bench","oneLine":"GAMA-Bench evaluates LLMs on gender-asymmetric moral framing across 1,298 paired conflict scenarios, measuring response differences in punitive, therapeutic, and blame dimensions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-12","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.14068","pdf":"https://arxiv.org/pdf/2606.14068","project":null,"code":"https://github.com/xufeiqiong/GAMA-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.14068"},"evidence":{"snippet":"We introduce GAMA-Bench, a gender-mirrored benchmark of 1,298 scenarios covering intimate relationship and public social conflicts.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":10,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.14068"},"ranking":{"90d":{"score":58,"rank":141,"coverage":0.55,"confidence":"Low"}},"description":"GAMA-Bench evaluates LLMs on gender-asymmetric moral framing across 1,298 paired conflict scenarios, measuring response differences in punitive, therapeutic, and blame dimensions.","whyItMatters":"detects biases beyond stereotypes by comparing responses to matched male and female actors, informing fairness in AI decision-making.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"61b891d419f1b1cd095b0519941dab27866e3aa7645a14710bdd707f759b7a1b"},"motivation":"Existing studies on gender bias in LLMs have largely focused on stereotypes, occupational associations, or explicit harmful outputs.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.14068","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_decompbench_f0f37f4f","familyId":"bmf_820b856726b1","name":"DeCompBench","oneLine":"DeCompBench evaluates agent safety against decomposition attacks, where harmful tasks are broken into benign subtasks; it measures refusal rates and objective fulfillment on decomposed variants.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13994","pdf":"https://arxiv.org/pdf/2606.13994","project":null,"code":null,"data":"https://huggingface.co/datasets/decompositionbench/DeCompBench","hfPaper":"https://huggingface.co/papers/2606.13994"},"evidence":{"snippet":"To this end, we introduce DeCompBench, a benchmark designed specifically to evaluate agentic safety under decomposition attacks.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":54,"hfDatasetLikes":2},"source":{"type":"arxiv","id":"2606.13994"},"ranking":{"90d":{"score":37,"rank":303,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":42,"datasetRankPopulation":51}},"description":"DeCompBench evaluates agent safety against decomposition attacks, where harmful tasks are broken into benign subtasks; it measures refusal rates and objective fulfillment on decomposed variants.","whyItMatters":"Addresses a security gap not covered by existing agent safety benchmarks, critical for preventing adversarial misuse in deployed agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"13f49fa85a55bff0af3e7ab967aab671bf9160d72b2b6ee6fbd1ecf49e0a55aa"},"motivation":"LLM-based Agents are becoming increasingly capable and widely deployed, creating growing incentives for adversarial misuse in the real-world.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13994","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_clinhallu_85f948a0","familyId":"bmf_52fa2852cf5f","name":"ClinHallu","oneLine":"ClinHallu is a benchmark for diagnosing stage-wise hallucinations in medical multimodal large language models. It contains 7,031 instances with structured reasoning traces decomposed into visual recognition, knowledge recall, and reasoning integration, along with stage-replacement interventions for measuring final answer changes.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.14697","pdf":"https://arxiv.org/pdf/2606.14697","project":null,"code":"https://github.com/alibaba-damo-academy/ClinHallu","data":null,"hfPaper":"https://huggingface.co/papers/2606.14697"},"evidence":{"snippet":"To enable source-level hallucination diagnosis, we introduce ClinHallu, a benchmark for stage-wise hallucination diagnosis in medical MLLM reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":"2026-06-15T00:00:00.000Z","githubStars":9,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.14697"},"ranking":{"90d":{"score":60,"rank":133,"coverage":0.7,"confidence":"Medium"}},"description":"ClinHallu is a benchmark for diagnosing stage-wise hallucinations in medical multimodal large language models. It contains 7,031 instances with structured reasoning traces decomposed into visual recognition, knowledge recall, and reasoning integration, along with stage-replacement interventions for measuring final answer changes.","whyItMatters":"Existing medical hallucination benchmarks often ignore the source of hallucinations within reasoning. ClinHallu allows for fine-grained diagnosis of where errors originate, providing a testbed for improving model reliability in clinical decision support.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c8e07fb367894c41ba360c6edec112e26ee266686dcf9cb604385753e9d977d2"},"motivation":"Building trustworthy medical multimodal large language models (MLLMs) is critical for reliable clinical decision support.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.14697","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Alibaba DAMO Academy","organizationType":"company-research-lab","sourceUrl":"https://github.com/alibaba-damo-academy/ClinHallu","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_affordance20q_4a1ecb35","familyId":"bmf_b583dac6d78c","name":"AFFORDANCE20Q","oneLine":"Affordance20Q is a benchmark for evaluating affordance reasoning in LLMs using a 20-questions game. It comprises 1,009 games over 454 objects and 59 affordances, where models identify a hidden object's affordance by asking yes/no questions about physical properties.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-12","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.14240","pdf":"https://arxiv.org/pdf/2606.14240","project":null,"code":"https://github.com/1171-jpg/Affordance20Q.git","data":null,"hfPaper":"https://huggingface.co/papers/2606.14240"},"evidence":{"snippet":"To address this gap, we introduce Affordance20Q, a novel affordance reasoning benchmark formulated as a 20-Questions game without exposing the object's identity.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":5,"hfDailySubmittedAt":"2026-06-15T00:00:00.000Z","githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.14240"},"ranking":{"90d":{"score":31,"rank":311,"coverage":0.7,"confidence":"Medium"}},"description":"Affordance20Q is a benchmark for evaluating affordance reasoning in LLMs using a 20-questions game. It comprises 1,009 games over 454 objects and 59 affordances, where models identify a hidden object's affordance by asking yes/no questions about physical properties.","whyItMatters":"Affordance reasoning is fundamental to physical understanding. Affordance20Q tests whether models can reason over physical properties without relying on memorized object-affordance mappings, which is crucial for embodied AI.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dd61f2b3753053a683d5b5fa8b53ec7f3847c5984b3afc0b06672e4172a4b092"},"motivation":"Affordance reasoning, the inference of an object's action possibilities from its physical properties (e.g., shape and material), is fundamental to human physical understanding and increasingly critical for Large Language Models (LLMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.14240","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_vietfashion_31e76260","familyId":"bmf_039d4698ef43","name":"VietFashion","oneLine":"VietFashion is a benchmark for sketch-text composed image retrieval centered on the Ao Dai, a traditional Vietnamese garment. It includes 650 sketches expanded to over 21,000 photorealistic images with captions, and adopts a multi-target retrieval setting.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13427","pdf":"https://arxiv.org/pdf/2606.13427","project":"https://hng0303.github.io/VietFashion","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.13427"},"evidence":{"snippet":"We introduce VietFashion, a new benchmark for sketch-text composed image retrieval centered on the Ao Dai, a traditional Vietnamese garment.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13427"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"VietFashion is a benchmark for sketch-text composed image retrieval centered on the Ao Dai, a traditional Vietnamese garment. It includes 650 sketches expanded to over 21,000 photorealistic images with captions, and adopts a multi-target retrieval setting.","whyItMatters":"Cultural garments require fine-grained retrieval systems that understand subtle structural and symbolic details. VietFashion exposes gaps in modeling cultural semantics and multi-modal composition.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"69626db0485cc05a310a0b2006a18317ac7387c1bb1737f741d0f33b2324ebe4"},"motivation":"Cultural garments pose a unique challenge for visual retrieval systems, as their identity often depends on subtle structural and symbolic details that are poorly captured by standard AI models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13427","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Search & Retrieval"],"domainScope":"general"},{"id":"bm_vhdlbench_8f92d226","familyId":"bmf_866c4b9ba052","name":"VHDLBench","oneLine":"VHDLSuite is a benchmark-centered infrastructure for VHDL generation evaluation, integrating automated benchmark synthesis, executable validation, and multi-model diagnostic analysis. It includes VHDLBench with over 200 VHDL problems with validated testbenches.","area":"Language & Knowledge","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Semiconductors"],"capabilities":[],"topics":["cs.AR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13735","pdf":"https://arxiv.org/pdf/2606.13735","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.13735"},"evidence":{"snippet":"Second, we introduce VHDLBench, a benchmark with over 200 VHDL problems with complete and validated testbenches across a wide range of complexity levels.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13735"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"VHDLSuite is a benchmark-centered infrastructure for VHDL generation evaluation, integrating automated benchmark synthesis, executable validation, and multi-model diagnostic analysis. It includes VHDLBench with over 200 VHDL problems with validated testbenches.","whyItMatters":"Evaluating LLM performance in VHDL generation is limited compared to Verilog. VHDLSuite provides a standardized pipeline for scalable VHDL evaluation, addressing distinct language characteristics.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d40def589edb329f931ed356414118606c52f8974e9d8862a8084b53a19f8cfe"},"motivation":"Large Language Models (LLM) have shown impressive capabilities in Register Transfer Level (RTL) code generation, particularly for Verilog.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13735","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_uoj-bench_c2013a7a","familyId":"bmf_e38a83758c16","name":"UOJ-Bench","oneLine":"UOJ-Bench is a benchmark for evaluating LLMs in code generation, hacking, and repair, built from real-world submissions on the Universal Online Judge and evaluated through UOJ's judging infrastructure.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation"],"topics":["Code"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.12864","pdf":"https://arxiv.org/pdf/2606.12864","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.12864"},"evidence":{"snippet":"In this work, we introduce UOJ-Bench, a benchmark designed to evaluate not only the problem-solving ability of LLMs, but also their ability to identify errors in human-written code -- a crucial educational activity traditionally supported by running test cases over online judge systems.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.12864"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"UOJ-Bench is a benchmark for evaluating LLMs in code generation, hacking, and repair, built from real-world submissions on the Universal Online Judge and evaluated through UOJ's judging infrastructure.","whyItMatters":"UOJ-Bench extends beyond problem-solving to include identifying errors in human code, a critical educational activity. It provides a realistic setting for assessing LLM support in learning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2f39985c58bc9171a708ac301f999dd18b2e1ba063f6730c1b45475f188de916"},"motivation":"Despite strong performance in competitive programming, the role of Large Language Models (LLMs) in supporting human learning in the same setting remains largely unexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.12864","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_terrabench_679be7e3","familyId":"bmf_b1de429d892f","name":"TerraBench","oneLine":"TerraBench is a benchmark for grounded Earth-science reasoning, built on TerraAgent, a ReAct-style framework that couples LLM planning with scientific tools. It includes 403 tasks across three tracks and eight domains with 24,500 verified steps.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Planning","Information retrieval"],"topics":["Reasoning"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13148","pdf":"https://arxiv.org/pdf/2606.13148","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.13148"},"evidence":{"snippet":"We introduce TerraBench, a benchmark for grounded Earth-science reasoning, built on TerraAgent, a ReAct-style executable framework that interleaves reasoning, tool calls, and observations to couple LLM planning with scientific tools for environmental retrieval, geospatial processing, simulation, and artifact-backed computation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13148"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TerraBench is a benchmark for grounded Earth-science reasoning, built on TerraAgent, a ReAct-style framework that couples LLM planning with scientific tools. It includes 403 tasks across three tracks and eight domains with 24,500 verified steps.","whyItMatters":"Earth-science workflows require reasoning over heterogeneous data types. TerraBench unifies these capabilities in a single interface, with process-level metrics and tolerance-aware scoring.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cf0c3dd97e1cf343dfa8a50d16094860427a3ca3115107919789d9da73088f21"},"motivation":"Climate and environmental decision-making increasingly requires reasoning across heterogeneous inputs, including gridded physical data, satellite imagery, geospatial context, and simulator outputs.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13148","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_stakebench_be4543bc","familyId":"bmf_bb61545ac7a2","name":"StakeBench","oneLine":"StakeBench is a stakeholder-centric benchmark for prompt-injection attacks in web agents for online shopping. It decomposes risk into 12 attack objectives across three stakeholder classes, with 264 adversarial cases across 12 product categories.","area":"Language & Knowledge","applicationDomains":["Cybersecurity","Finance & Economics"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity","Financial Services"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13385","pdf":"https://arxiv.org/pdf/2606.13385","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.13385"},"evidence":{"snippet":"To capture these properties, we introduce StakeBench, a stakeholder-centric benchmark that systematically categorizes and attributes harm in real-world web agent systems for online shopping.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13385"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"StakeBench is a stakeholder-centric benchmark for prompt-injection attacks in web agents for online shopping. It decomposes risk into 12 attack objectives across three stakeholder classes, with 264 adversarial cases across 12 product categories.","whyItMatters":"Prompt-injection risk is victim-dependent. StakeBench captures asymmetric consequences for different stakeholders, which is overlooked by attack-centric evaluations.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ce4c91daad8cf70bdc63538671da0ef1118b36aa9229478867c967e28a5752e2"},"motivation":"LLM-based web agents are increasingly deployed in real-world settings such as e-commerce, where they interact extensively with untrusted web content while executing actions that carry direct financial consequences.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13385","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"cross-domain"},{"id":"bm_sevra-bench_b52ddf44","familyId":"bmf_74c5a4c5c66f","name":"SEVRA-BENCH","oneLine":"SEVRA-BENCH is a benchmark for measuring how often LLM-based code review agents approve adversarial pull requests with social-engineering framings, built from vulnerability-fixing commits. It includes a challenge split of roughly 1,000 adversarial PRs.","area":"Language & Knowledge","applicationDomains":["Cybersecurity","Robotics & Autonomous Systems"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity","Robotics"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.13757","pdf":"https://arxiv.org/pdf/2606.13757","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.13757"},"evidence":{"snippet":"We introduce SEVRA-BENCH (Social Engineering of Vulnerabilities in Review Agents), a benchmark that measures how often a review agent approves such adversarial PR s.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13757"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SEVRA-BENCH is a benchmark for measuring how often LLM-based code review agents approve adversarial pull requests with social-engineering framings, built from vulnerability-fixing commits. It includes a challenge split of roughly 1,000 adversarial PRs.","whyItMatters":"Review agents are susceptible to narrative manipulation, which can lead to merging vulnerable code. SEVRA-BENCH quantifies this gap in security capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f38cfe0a8590ef58503520196d8568252bb886e6eb913e6af5d30bbfbc991af0"},"motivation":"Large language models (LLMs) are increasingly deployed in automated code-review systems, where their approvals can determine which code is merged into shared repositories.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13757","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"cross-domain"},{"id":"bm_scir_5305af40","familyId":"bmf_fdd9b59047b7","name":"SciR","oneLine":"SciR evaluates LLMs on deduction, induction, and causal abduction in scientific settings, with tasks generated from formal objects and rendered into multi-document scientific discourse. Difficulty is controlled along extraction and inference axes, with verifiable answers.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13020","pdf":"https://arxiv.org/pdf/2606.13020","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.13020"},"evidence":{"snippet":"We introduce SciR, a benchmark that combines multi-paradigm reasoning with controllable scientific rendering, anchored on three paradigmatic scientific problems.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13020"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SciR evaluates LLMs on deduction, induction, and causal abduction in scientific settings, with tasks generated from formal objects and rendered into multi-document scientific discourse. Difficulty is controlled along extraction and inference axes, with verifiable answers.","whyItMatters":"Existing benchmarks either lack mechanistic ground truth or do not resemble real scientific documents. SciR provides a controllable protocol for isolating extraction vs. inference failures, which is valuable for diagnosing model capabilities in scientific reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3e4c1dad6af2a8ed0bc0ad7f976f06ef9e4ea462214a544d574a00883e5f946d"},"motivation":"Three paradigmatic forms of inference recur across scientific reasoning: deduction, induction, and causal abduction.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13020","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_roboprocessbench_629af272","familyId":"bmf_51d8097c561d","name":"RoboProcessBench","oneLine":"RoboProcessBench evaluates vision-language models on process-aware understanding in robotic manipulation, with 12 diagnostic question families covering static monitoring and dynamic reasoning over execution traces. The benchmark includes 58k QA pairs across 260 tasks.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics","Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13040","pdf":"https://arxiv.org/pdf/2606.13040","project":"https://processbench-2026.github.io/RoboProcessBench-Web/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.13040"},"evidence":{"snippet":"To address this gap, we present RoboProcessBench, a benchmark for process-aware understanding in vision-language robotic manipulation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13040"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"RoboProcessBench evaluates vision-language models on process-aware understanding in robotic manipulation, with 12 diagnostic question families covering static monitoring and dynamic reasoning over execution traces. The benchmark includes 58k QA pairs across 260 tasks.","whyItMatters":"Existing evaluations largely ignore fine-grained process understanding, which is crucial for VLMs used as critics or failure detectors. This benchmark provides a structured way to measure progress in this capability and supports post-training via a dedicated SFT split.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9967320f876bc6201dd5f2694cb322a41863de83122a184d7a74dcd26496a1e0"},"motivation":"Vision-language models (VLMs) are increasingly explored as visual critics, reward generators, and failure detectors in robotic manipulation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13040","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"RoboProcessBench Project","organizationType":"community","sourceUrl":"https://processbench-2026.github.io/RoboProcessBench-Web/","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_phantomfill_4f8c991e","familyId":"bmf_c5647cf4fd51","name":"PhantomFill","oneLine":"PhantomFill measures schema-coerced fabrication in language models by asking questions on unanswerable inputs under three output formats: free text, JSON with an escape option, and JSON with required fields. It reports Coerced Fabrication Rate and Escape Utilization Rate.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20492","pdf":"https://arxiv.org/pdf/2607.20492","project":null,"code":"https://github.com/ranausmanai/phantomfill","data":null,"hfPaper":"https://huggingface.co/papers/2607.20492"},"evidence":{"snippet":"We release PhantomFill, a benchmark with deterministic scoring and two reportable numbers: the Coerced Fabrication Rate and the Escape Utilization Rate.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20492"},"ranking":{"90d":{"score":29,"rank":382,"coverage":0.7,"confidence":"Medium"}},"description":"PhantomFill measures schema-coerced fabrication in language models by asking questions on unanswerable inputs under three output formats: free text, JSON with an escape option, and JSON with required fields. It reports Coerced Fabrication Rate and Escape Utilization Rate.","whyItMatters":"Hallucination in form-filling contexts is under-measured and costly. PhantomFill provides deterministic, code-based metrics targeting a critical failure mode, enabling model comparison and safety evaluation in structured output settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dc0cfa8ee8155946c8493880bd83e29bc0d25bb525a41a8bf1625cd50dde239c"},"motivation":"Language models in production do not write prose.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20492","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_mlubench_e20fa573","familyId":"bmf_8d9e10142c64","name":"MLUBench","oneLine":"MLUBench evaluates multimodal large language models on lifelong unlearning across 127 entities and 9 classes, providing QA pairs and images. It includes a protocol for sequential unlearning requests and evaluation of forgetting and retention.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.12809","pdf":"https://arxiv.org/pdf/2606.12809","project":null,"code":"https://github.com/lihe-maxsize/Lifelong_Unlearning_main","data":null,"hfPaper":"https://huggingface.co/papers/2606.12809"},"evidence":{"snippet":"To fill this gap, we introduce the MLUBench, a large-scale and comprehensive benchmark featuring 127 entities across 9 classes under lifelong unlearning requests.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.12809"},"ranking":{"90d":{"score":39,"rank":271,"coverage":0.7,"confidence":"Medium"}},"description":"MLUBench evaluates multimodal large language models on lifelong unlearning across 127 entities and 9 classes, providing QA pairs and images. It includes a protocol for sequential unlearning requests and evaluation of forgetting and retention.","whyItMatters":"Lifelong unlearning is a practical challenge for MLLMs as data removal requests arrive over time. This benchmark enables systematic evaluation of unlearning methods and highlights the unique constraint of preserving multimodal alignment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"02c0e16073f9c2b6fad8fb9dd7b135c9ffd55336756967d2eb09fd92d89be05d"},"motivation":"Multimodal large language models (MLLMs) are trained on massive multimodal data, making data unlearning increasingly important as data owners may request the removal of specific content.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026","evidence":"36 pages, accepted to the ICML 2026","evidenceUrl":"https://arxiv.org/abs/2606.12809","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026","reviewStatus":"accepted","decisionRaw":"36 pages, accepted to the ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.12809","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"36 pages, accepted to the ICML 2026","level":"author-claim"}]}],"publishers":[{"name":"MLUBench Project","organizationType":"community","sourceUrl":"https://github.com/lihe-maxsize/Lifelong_Unlearning_main","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_ma-proofbench_ec795126","familyId":"bmf_2dea3b94ab49","name":"MA-ProofBench","oneLine":"MA-ProofBench evaluates LLMs on theorem proving in mathematical analysis using 200 Lean 4 formalized problems split into undergraduate and Ph.D. levels. It covers six core topics and uses Pass@8 with formal verification via Lean server.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.13782","pdf":"https://arxiv.org/pdf/2606.13782","project":null,"code":"https://github.com/OpenBMB/MA-ProofBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.13782"},"evidence":{"snippet":"To address this gap, we introduce MA-ProofBench, to the best of our knowledge, the first formal theorem-proving benchmark dedicated to Mathematical Analysis.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13782"},"ranking":{"90d":{"score":47,"rank":213,"coverage":0.7,"confidence":"Medium"}},"description":"MA-ProofBench evaluates LLMs on theorem proving in mathematical analysis using 200 Lean 4 formalized problems split into undergraduate and Ph.D. levels. It covers six core topics and uses Pass@8 with formal verification via Lean server.","whyItMatters":"Formal theorem proving benchmarks typically lack coverage of advanced analysis. MA-ProofBench fills this gap, providing a stable evaluation for tracking progress in this difficult domain.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"89011505fc7da03fb120410278f5d7b8b383714e7b448165af92d4a4f07f5858"},"motivation":"Large Language Models (LLMs) have made notable progress in automated theorem proving, yet existing formal benchmarks remain limited in both mathematical coverage and difficulty.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13782","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"OpenBMB","organizationType":"community","sourceUrl":"https://github.com/OpenBMB/MA-ProofBench","role":"benchmark-publisher"}],"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_geonatureagent-benchmark_854d9ae1","familyId":"bmf_245d2d78860d","name":"GeoNatureAgent Benchmark","oneLine":"GeoNatureAgent Benchmark evaluates LLM agents on environmental geospatial analysis through structured tool calls to a self-hostable API. It includes 93 tasks across 18 categories, with metrics for accuracy and cost.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.12821","pdf":"https://arxiv.org/pdf/2606.12821","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.12821"},"evidence":{"snippet":"We introduce the GeoNatureAgent Benchmark, the first benchmark for environmental analysis agents that operate via structured tool calls to a production-style geospatial API.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.12821"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GeoNatureAgent Benchmark evaluates LLM agents on environmental geospatial analysis through structured tool calls to a self-hostable API. It includes 93 tasks across 18 categories, with metrics for accuracy and cost.","whyItMatters":"There is a lack of benchmarks for agentic geospatial workflows, and this benchmark provides a realistic API-based environment, enabling assessment of tool-use reasoning and cost-efficiency trade-offs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"258c8d828d24cdc11b7d02349ed26a10fff958e2f5deea883d5a0db021929c5a"},"motivation":"Environmental scientists spend disproportionate effort on data wrangling rather than analysis, and AI agents that automate geospatial workflows remain unvalidated: no benchmark evaluates agents operating through structured tool calling against real APIs.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.12821","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_evobrowsecomp_ef2e05ce","familyId":"bmf_482f8121203a","name":"EvoBrowseComp","oneLine":"EvoBrowseComp evaluates search agents on evolving knowledge with 800 contamination-free questions synthesized from live web traversal. The benchmark is designed to be regularly updated to prevent contamination.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13120","pdf":"https://arxiv.org/pdf/2606.13120","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.13120"},"evidence":{"snippet":"In this paper, we introduce EvoBrowseComp, an evolving benchmark of 400 English and 400 Chinese contamination-free complex questions synthesized via live-web traversal.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":4,"hfDailySubmittedAt":"2026-06-12T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13120"},"ranking":{"90d":{"score":52,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"EvoBrowseComp evaluates search agents on evolving knowledge with 800 contamination-free questions synthesized from live web traversal. The benchmark is designed to be regularly updated to prevent contamination.","whyItMatters":"Static benchmarks suffer from contamination and memorization, obscuring genuine retrieval. EvoBrowseComp offers a dynamic, auto-updated evaluation, which is critical for measuring true browsing competence.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"15444e18d15747256fc0896ef09365d4f74f1c087ba5ad1c72b655a0b2ccacd6"},"motivation":"Search Agents -- large language models augmented with search tools -- have intensified the need for future-proof evaluation benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13120","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_evoarena_70b33a0d","familyId":"bmf_75e1fa5bb7d6","name":"EvoArena","oneLine":"EvoArena is a benchmark suite for LLM agents in dynamic environments, covering terminal workflows, software repositories, and social preferences. It evaluates step and chain accuracy under progressive environment updates.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13681","pdf":"https://arxiv.org/pdf/2606.13681","project":null,"code":"https://github.com/Aiden0526/EvoArena","data":null,"hfPaper":"https://huggingface.co/papers/2606.13681"},"evidence":{"snippet":"To address this gap, we introduce EvoArena, a benchmark suite that models environment changes as sequences of progressive updates across terminal, software, and social domains.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":143,"hfDailySubmittedAt":"2026-06-12T00:00:00.000Z","githubStars":23,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13681"},"ranking":{"90d":{"score":72,"rank":67,"coverage":0.7,"confidence":"Medium"}},"description":"EvoArena is a benchmark suite for LLM agents in dynamic environments, covering terminal workflows, software repositories, and social preferences. It evaluates step and chain accuracy under progressive environment updates.","whyItMatters":"Real-world deployments are dynamic, and existing benchmarks often ignore environment evolution. EvoArena provides a protocol for evaluating agent adaptation and highlights the need for memory models that track changes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b867bf3a4511c256aa422e7d59d8b6e44ef587e04b2a370e8d4b6bb124f6271c"},"motivation":"Large language model (LLM) agents have achieved strong performance on a wide range of benchmarks, yet most evaluations assume static environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13681","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"EvoArena Project","organizationType":"community","sourceUrl":"https://github.com/Aiden0526/EvoArena","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_epibench_be78266a","familyId":"bmf_12b5a09e9b7d","name":"EpiBench","oneLine":"EpiBench evaluates AI agents on short-horizon epigenomics analysis tasks across CUT&Tag/CUT&RUN, ATAC-seq, ChIP-seq, and DNA methylation workflows, with deterministically gradable answers.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13602","pdf":"https://arxiv.org/pdf/2606.13602","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.13602"},"evidence":{"snippet":"We introduce EpiBench, a verifiable benchmark for short-horizon epigenomics analysis.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13602"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EpiBench evaluates AI agents on short-horizon epigenomics analysis tasks across CUT&Tag/CUT&RUN, ATAC-seq, ChIP-seq, and DNA methylation workflows, with deterministically gradable answers.","whyItMatters":"It addresses the lack of verifiable benchmarks for epigenomics analysis agents, providing a basis for comparing agent performance on complex scientific tasks that require domain-specific judgment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"034189a803fb4170a9fc7ed058dbf2356b41f6195ef3fa0f3290efd8c59bea8b"},"motivation":"We introduce EpiBench, a verifiable benchmark for short-horizon epigenomics analysis.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13602","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_dlawbench_4866c9b4","familyId":"bmf_48d9ddc30394","name":"DLawBench","oneLine":"DLawBench evaluates multi-turn legal consultation in Chinese and U.S. law across four client personas. It scores LLMs on information gathering, legal reasoning, and claim support using 461 cases, 5,532 paired fact entries, inquiry and issue rubrics, and a public leaderboard.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13931","pdf":"https://arxiv.org/pdf/2606.13931","project":null,"code":"https://github.com/SKYLENAGE-AI/DLawBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.13931"},"evidence":{"snippet":"To fill this gap, we introduce DLawBench, a diagnostic benchmark for real-world legal consultation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13931"},"ranking":{"90d":{"score":46,"rank":222,"coverage":0.55,"confidence":"Low"}},"description":"DLawBench evaluates multi-turn legal consultation in Chinese and U.S. law across four client personas. It scores LLMs on information gathering, legal reasoning, and claim support using 461 cases, 5,532 paired fact entries, inquiry and issue rubrics, and a public leaderboard.","whyItMatters":"Existing legal benchmarks assume complete fact patterns, overlooking the interactive elicitation needed in real consultations. DLawBench provides a diagnostic evaluation of LLM capability in legal consultation, revealing performance gaps and failure modes that inform development of models for legal assistance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2c69f1f9a9969aa88bb2f102fa252169159f7fd5b0cc104ee8b1b1d068085f0e"},"motivation":"Lawyer-client consultation is a critical starting point for legal services.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13931","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SKYLENAGE-AI","organizationType":"community","sourceUrl":"https://github.com/SKYLENAGE-AI/DLawBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_dailyreport_bfd6c726","familyId":"bmf_5bf51bae2f57","name":"DailyReport","oneLine":"DailyReport is a benchmark with 150 open-ended daily search tasks and 3,546 rubrics, evaluating search agents on information-seeking tasks. Tasks are decomposed into subtasks with cascade rubrics across dimensions, yielding interpretable scores and a user preference score.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.12871","pdf":"https://arxiv.org/pdf/2606.12871","project":null,"code":"https://github.com/AGI-Eval-Official/DailyReport","data":null,"hfPaper":"https://huggingface.co/papers/2606.12871"},"evidence":{"snippet":"To bridge this gap, we introduce DailyReport, an open-ended benchmark to evaluate SA capabilities on daily search tasks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":14,"hfDailySubmittedAt":"2026-06-23T00:00:00.000Z","githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.12871"},"ranking":{"90d":{"score":58,"rank":146,"coverage":0.7,"confidence":"Medium"}},"description":"DailyReport is a benchmark with 150 open-ended daily search tasks and 3,546 rubrics, evaluating search agents on information-seeking tasks. Tasks are decomposed into subtasks with cascade rubrics across dimensions, yielding interpretable scores and a user preference score.","whyItMatters":"Existing search agent benchmarks often use specialized or artificial tasks with coarse scoring, limiting interpretability and real-world relevance. DailyReport provides a more user-centric evaluation protocol for daily search tasks, offering fine-grained, dimension-level scores to inform agent development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b8f41d9137f5c7ed7c1d1328acc7e7a8ba8527f8f34d6acc08bc0cc6a4f0f74c"},"motivation":"Search Agents (SAs) typically leverage large language models (LLMs) to support complex information-seeking tasks by autonomously exploring web sources and synthesizing information into comprehensive responses.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.12871","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_contactworld_92ad6aa1","familyId":"bmf_0a6612dbc958","name":"ContactWorld","oneLine":"ContactWorld is a benchmark and empirical study for vision-tactile world models in contact-rich manipulation, spanning 12 tasks including insertion and screwing, evaluating planning success rates.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13877","pdf":"https://arxiv.org/pdf/2606.13877","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.13877"},"evidence":{"snippet":"In this paper, we present ContactWorld, a benchmark and systematic empirical study of vision-tactile world models spanning 12 contact-rich manipulation tasks, including insertion, disassembly, screwing, and exploratory interaction.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13877"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"ContactWorld is a benchmark and empirical study for vision-tactile world models in contact-rich manipulation, spanning 12 tasks including insertion and screwing, evaluating planning success rates.","whyItMatters":"Addresses the gap in understanding which representation properties support stable long-horizon planning in contact-rich settings, offering practical guidance for multimodal model design.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"61ea2b88097ea2e06dd79d50d405a9177767e5ff1c2e893c2d32f1d945e3cfc4"},"motivation":"Contact-rich manipulation requires world models to reason over complex contact dynamics from multimodal sensory observations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.13877","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_cloudcons_9f1d2495","familyId":"bmf_7839980df73c","name":"CloudCons","oneLine":"CloudCons is an end-to-end benchmark for evaluating forecasting models in cloud resource consolidation. It encompasses datasets from Huawei Cloud, Microsoft Azure, and Google Borg with diverse workload patterns, and evaluates statistical, deep learning, and time series foundation models. The evaluation includes resource efficiency and service reliability metrics under a forecast-then-optimize paradigm.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13513","pdf":"https://arxiv.org/pdf/2606.13513","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.13513"},"evidence":{"snippet":"To bridge this gap, we propose CloudCons, a comprehensive end-to-end benchmark designed to evaluate forecasting models within the specific context of cloud resource consolidation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13513"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CloudCons is an end-to-end benchmark for evaluating forecasting models in cloud resource consolidation. It encompasses datasets from Huawei Cloud, Microsoft Azure, and Google Borg with diverse workload patterns, and evaluates statistical, deep learning, and time series foundation models. The evaluation includes resource efficiency and service reliability metrics under a forecast-then-optimize paradigm.","whyItMatters":"Existing benchmarks focus only on prediction error, leaving the downstream decision utility of forecasting models unverified. CloudCons addresses this gap by assessing practical value in cloud resource consolidation, offering insights into balancing resource efficiency and service reliability, aiding deployment decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cef34bc8597b5f95de0aff24a0814fcb91b64cd035c81390cd5da7407290cd36"},"motivation":"Driven by conservative over-provisioning to guarantee service reliability, resource utilization in cloud data centers remains at low levels.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"KDD 2026","evidence":"Accepted to KDD 2026","evidenceUrl":"https://arxiv.org/abs/2606.13513","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"KDD 2026","reviewStatus":"accepted","decisionRaw":"Accepted to KDD 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.13513","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to KDD 2026","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_authoritybench_5d25d571","familyId":"bmf_fba66cb2685f","name":"AuthorityBench","oneLine":"AuthorityBench evaluates how citation-based authority signals affect epistemic behavior in LLMs using 220,564 prompts across four domains, with a 2x2 factorial design crossing claim veracity and citation veracity.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-11","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.13104","pdf":"https://arxiv.org/pdf/2606.13104","project":null,"code":"https://github.com/floating-reeds/AuthorityBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.13104"},"evidence":{"snippet":"We introduce AuthorityBench, a 220,564-prompt multi-domain benchmark that isolates how citation-based authority signals influence epistemic behavior in LLMs.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.13104"},"ranking":{"90d":{"score":29,"rank":376,"coverage":0.55,"confidence":"Low"}},"description":"AuthorityBench evaluates how citation-based authority signals affect epistemic behavior in LLMs using 220,564 prompts across four domains, with a 2x2 factorial design crossing claim veracity and citation veracity.","whyItMatters":"The benchmark isolates citation presence from content to measure susceptibility to citation-induced hallucination, providing a standardized protocol for assessing reliability in citation-augmented settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cc87a45f373a49769ef8548228e8198ada140e0c2bb0e4c064a8cf4fd98198d3"},"motivation":"Large language models are increasingly deployed in citation-augmented settings, yet the effect of citation presence on model behavior independent of factual content remains poorly understood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"AI4GOOD and EIML at ICML 2026","evidence":"10 pages, 5 figures. Accepted to AI4GOOD and EIML at ICML 2026","evidenceUrl":"https://arxiv.org/abs/2606.13104","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"AI4GOOD and EIML at ICML 2026","reviewStatus":"accepted","decisionRaw":"10 pages, 5 figures. Accepted to AI4GOOD and EIML at ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.13104","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"10 pages, 5 figures. Accepted to AI4GOOD and EIML at ICML 2026","level":"author-claim"}]}],"publishers":[{"name":"floating-reeds","organizationType":"community","sourceUrl":"https://github.com/floating-reeds/AuthorityBench","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_shopping-reasoning-bench_1ec16882","familyId":"bmf_241bcbe846af","name":"Shopping Reasoning Bench","oneLine":"Assesses multi-turn conversational shopping assistants across 525 expert-authored missions with importance-weighted binary rubrics. Evaluates reasoning across five categories covering preference refinement, trade-off analysis, and compatibility assessment.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-10","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.12608","pdf":"https://arxiv.org/pdf/2606.12608","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.12608"},"evidence":{"snippet":"We introduce the Shopping Reasoning Bench, an expert-authored benchmark of 525 missions (232 single-turn, 293 multi-turn) with 10863 importance-weighted binary rubrics authored by retail domain experts.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.12608"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Assesses multi-turn conversational shopping assistants across 525 expert-authored missions with importance-weighted binary rubrics. Evaluates reasoning across five categories covering preference refinement, trade-off analysis, and compatibility assessment.","whyItMatters":"Fills the lack of benchmarks for open-ended shopping dialogue with objective criteria and nuanced reasoning, providing a standardized testbed for improving assistant performance in real-world e-commerce.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"55844d5eb57bd09832f19037d0227541a00378d75d1ac069ce01f7cc260e7955"},"motivation":"Conversational shopping assistants now serve hundreds of millions of customers, yet no existing benchmark jointly evaluates the open-ended multi-turn reasoning, domain expertise, and criterion-level quality that real shopping conversations demand.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.12608","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Shopping Reasoning Bench Team","organizationType":"academic-lab","sourceUrl":"https://huggingface.co/papers/2606.12608","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_salart-vqa_f96d6eb3","familyId":"bmf_22505c6b720b","name":"SalArt-VQA","oneLine":"SalArt-VQA is a closed-set visual question answering benchmark for fine-grained salient artifact understanding in AI-generated images. It includes 950 images and 3,681 human-authored multiple-choice questions covering artifact images, matched real references, and paired generated references. Four aligned question types evaluate presence detection, semantic localization, spatial grounding, and evidence-grounded defect identification, with reference splits for calibration and abstention testing.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.12671","pdf":"https://arxiv.org/pdf/2606.12671","project":null,"code":null,"data":"https://huggingface.co/datasets/salartvqa/SalArt-VQA","hfPaper":"https://huggingface.co/papers/2606.12671"},"evidence":{"snippet":"To evaluate these behaviors directly, we introduce SalArt-VQA, a diagnostic benchmark for fine-grained SALient ARTifact understanding in AI-generated images.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":143,"hfDatasetLikes":2},"source":{"type":"arxiv","id":"2606.12671"},"ranking":{"90d":{"score":49,"rank":198,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":27,"datasetRankPopulation":51}},"description":"SalArt-VQA is a closed-set visual question answering benchmark for fine-grained salient artifact understanding in AI-generated images. It includes 950 images and 3,681 human-authored multiple-choice questions covering artifact images, matched real references, and paired generated references. Four aligned question types evaluate presence detection, semantic localization, spatial grounding, and evidence-grounded defect identification, with reference splits for calibration and abstention testing.","whyItMatters":"Image-level artifact detection accuracy can conceal failures in grounding and evidence use. This benchmark provides a fine-grained evaluation protocol that isolates specific failure modes, enabling comparison of VLMs on their ability to support artifact claims with local visual evidence. It offers practical value for developers selecting models for trustworthy artifact analysis in generated image workflows.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"86c84fb08aa08659e330506ba3665d09266cff50f02d60d2a9f0bf8f8c6e6408"},"motivation":"Vision-language models (VLMs) are increasingly used to detect whether AI-generated images contain visible artifacts, yet their ability to analyze such artifacts remains poorly understood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.12671","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SalArt-VQA Benchmark Team","organizationType":"community","sourceUrl":"https://huggingface.co/datasets/salartvqa/SalArt-VQA","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_rq-bench_0360bc38","familyId":"bmf_c9112d4b22f3","name":"RQ-Bench","oneLine":"RQ-Bench evaluates novelty of research questions generated by LLMs against author-anchored reference questions from recent arXiv papers, using standalone and comparative LLM judging as well as human expert evaluation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.DL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-10","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.12071","pdf":"https://arxiv.org/pdf/2606.12071","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.12071"},"evidence":{"snippet":"We introduce RQ-Bench, a benchmark built from recent arXiv papers.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":"2026-06-10T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.12071"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"RQ-Bench evaluates novelty of research questions generated by LLMs against author-anchored reference questions from recent arXiv papers, using standalone and comparative LLM judging as well as human expert evaluation.","whyItMatters":"Addresses the reliability of LLM-based novelty assessment for scientific ideation, indicating that LLM judges may produce a 'novelty mirage' compared to human experts. Useful for researchers evaluating automated scientific review or generation systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8e03f9b7c9de33ce9f875f99b46d0b781095724daff49d7d062b1b2e5c6ba26a"},"motivation":"LLMs are increasingly used to generate and judge scientific ideas.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.12071","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_medcta_84809bbe","familyId":"bmf_f4b829da8324","name":"MedCTA","oneLine":"MedCTA evaluates clinical tool agents on 107 clinician-verified, step-implicit tasks with multimodal inputs (radiology images, pathology slides, reports) and 5 deployed tools. Metrics cover tool selection, argument validity, execution stability, trajectory fidelity, and outcome quality.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11702","pdf":"https://arxiv.org/pdf/2606.11702","project":"https://ivul-kaust.github.io/MedCTA/","code":"https://github.com/IVUL-KAUST/MedCTA","data":"https://huggingface.co/datasets/IVUL-KAUST/MedCTA","hfPaper":"https://huggingface.co/papers/2606.11702"},"evidence":{"snippet":"We introduce MedCTA, a benchmark for evaluating medical tool agents on clinician-validated, step-implicit tasks grounded in realistic multimodal clinical inputs, including radiology images, pathology slides, and reports.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":438,"hfDatasetLikes":2},"source":{"type":"arxiv","id":"2606.11702"},"ranking":{"90d":{"score":60,"rank":136,"coverage":0.85,"confidence":"High","datasetDownloadRank":12,"datasetRankPopulation":51}},"description":"MedCTA evaluates clinical tool agents on 107 clinician-verified, step-implicit tasks with multimodal inputs (radiology images, pathology slides, reports) and 5 deployed tools. Metrics cover tool selection, argument validity, execution stability, trajectory fidelity, and outcome quality.","whyItMatters":"Fills a gap in medical AI evaluation by going beyond single-turn QA to test agentic tool use, planning, and reliability in real-world clinical workflows. Useful for auditing and advancing trustworthy medical AI agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1d3af7081fe9ba9810369fee5a9b4369e7e263d419ecf502a88211a080190291"},"motivation":"To make clinically grounded decisions, medical AI agents are expected to go beyond simple recognition and be capable of tool retrieval, evidence acquisition, and integration.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11702","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"IVUL-KAUST","organizationType":"academic-lab","sourceUrl":"https://ivul-kaust.github.io/MedCTA/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_hybridcodeauthorship_7e92f56f","familyId":"bmf_bcaf12963fb7","name":"HybridCodeAuthorship","oneLine":"HybridCodeAuthorship is a benchmark of Python files with interleaved human- and AI-authored lines, built from CodeSearchNet, for developing and evaluating AI-generated code detection algorithms at line- and chunk-level.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.12620","pdf":"https://arxiv.org/pdf/2606.12620","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.12620"},"evidence":{"snippet":"To fill these gaps, we introduce HybridCodeAuthorship, a novel benchmark of Python code files with interleaved human- and AI-authored lines of code to simulate authentic utilization of AI code assistants.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.12620"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HybridCodeAuthorship is a benchmark of Python files with interleaved human- and AI-authored lines, built from CodeSearchNet, for developing and evaluating AI-generated code detection algorithms at line- and chunk-level.","whyItMatters":"Addresses the need for realistic benchmarks for detecting AI-generated code in industry codebases, which is important for risk management and productivity analysis. Provides a challenging testbed for detection algorithms.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fc855bb09779b5652619bf4db361935122f3f53543be6fad8ea2cacec60c9b9c"},"motivation":"Thanks to the rapid adoption of AI code assistants powered by large language models (LLMs), industry codebases are, increasingly, a hybrid of AI- and human-authored code.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"publication_reported","venue":"LREC 2026 proceedings (pp. 1520-1532)","evidence":"LREC 2026 proceedings (pp. 1520-1532)","evidenceUrl":"https://arxiv.org/abs/2606.12620","source":"arxiv-journal-reference","evidenceLevel":"strong-author-metadata","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publications":[{"venueName":"LREC 2026 proceedings (pp. 1520-1532)","publicationStatus":"published","evidence":[{"sourceType":"arxiv-journal-reference","sourceUrl":"https://arxiv.org/abs/2606.12620","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"LREC 2026 proceedings (pp. 1520-1532)","level":"strong-author-metadata"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_graphinfer-bench_ae7fff34","familyId":"bmf_be514e0e285e","name":"GraphInfer-Bench","oneLine":"GraphInfer-Bench evaluates LLMs on graph inference tasks where answers reside in no single node or path, covering five task types over six real-world graphs with 42,000 samples. Tasks include masked-node prediction, edge inference, theme description, outlier detection, and community partition.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11562","pdf":"https://arxiv.org/pdf/2606.11562","project":null,"code":"https://github.com/graphinfer/GraphInfer-Bench","data":"https://huggingface.co/datasets/graphinfer/graphinfer","hfPaper":"https://huggingface.co/papers/2606.11562"},"evidence":{"snippet":"We introduce GraphInfer-Bench, a benchmark for whether LLMs can perform this graph inference: producing an open-ended answer that no single node supports and no path retrieves.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":"benchmark_repo","hfDatasetDownloads":60,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2606.11562"},"ranking":{"90d":{"score":43,"rank":251,"coverage":0.3,"confidence":"Low","datasetDownloadRank":38,"datasetRankPopulation":51}},"description":"GraphInfer-Bench evaluates LLMs on graph inference tasks where answers reside in no single node or path, covering five task types over six real-world graphs with 42,000 samples. Tasks include masked-node prediction, edge inference, theme description, outlier detection, and community partition.","whyItMatters":"Targets an open capability gap in graph understanding: inference over joint neighborhood structure. Useful for diagnosing weaknesses in LLMs and GNNs for tasks like fraud detection, drug repurposing, and recommendation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"29792f151b116d3a20164d719648aac9e29b1499b1b1a65d41dbeef11782020c"},"motivation":"Graph analysis underlies many applications whose answers cannot be looked up in a single record or retrieved along a path: laundering rings, drug repurposing, user preference, and scientific theme are all inferred from a node together with its neighbourhood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11562","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GraphInfer-Bench team","organizationType":"community","sourceUrl":"https://github.com/graphinfer/GraphInfer-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_extremewhenbench_bffe4798","familyId":"bmf_ee5095f4a257","name":"ExtremeWhenBench","oneLine":"ExtremeWhenBench evaluates natural-language temporal grounding in hour-long videos, with 2,273 open-form queries over 194 videos. Metrics include mIoU and R@k for predicted time intervals, with strict parse-failure handling.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.12300","pdf":"https://arxiv.org/pdf/2606.12300","project":null,"code":"https://github.com/naver-ai/ExtremeWhenBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.12300"},"evidence":{"snippet":"To test this, we release ExtremeWhenBench, the first open hour-scale grounding benchmark (2,273 queries over 194 videos, mean 75.7 min, max 9 hr) with an open-form query distribution.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":7,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.12300"},"ranking":{"90d":{"score":55,"rank":168,"coverage":0.7,"confidence":"Medium"}},"description":"ExtremeWhenBench evaluates natural-language temporal grounding in hour-long videos, with 2,273 open-form queries over 194 videos. Metrics include mIoU and R@k for predicted time intervals, with strict parse-failure handling.","whyItMatters":"Provides the first open hour-scale grounding benchmark to study the search problem in video understanding. Useful for developing and evaluating models for long-video retrieval and reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"84a01675c6e45d379bc29c233e8795d42c78d7afbeaed6f9da63114f6a1a4248"},"motivation":"Temporal grounding--returning the interval $[t_s, t_e]$ for a natural-language query over a video--is the language interface to long-form video, yet has been studied on short videos; the dynamics of hour-scale natural-language grounding remain underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.12300","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"NAVER AI","organizationType":"company-research-lab","sourceUrl":"https://github.com/naver-ai/ExtremeWhenBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_duobench_6c56fbaf","familyId":"bmf_bd016e251030","name":"DuoBench","oneLine":"DuoBench is a benchmarking framework for bimanual manipulation policies on the FR3 Duo platform, with eleven tasks across four coordination categories, simulated and partially real-world, plus human-teleoperated datasets. It proposes stage-based evaluation beyond binary success.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11901","pdf":"https://arxiv.org/pdf/2606.11901","project":"https://duobench.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.11901"},"evidence":{"snippet":"We introduce DuoBench, an extensible benchmarking framework for bimanual manipulation policies on the FR3 Duo platform.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.11901"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"DuoBench is a benchmarking framework for bimanual manipulation policies on the FR3 Duo platform, with eleven tasks across four coordination categories, simulated and partially real-world, plus human-teleoperated datasets. It proposes stage-based evaluation beyond binary success.","whyItMatters":"Provides a reproducible testbed for diagnosing failures in dual-arm policy learning, covering coordination challenges not captured by existing benchmarks. Useful for robotics researchers working on bimanual manipulation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"91b39f668a6ea3e7139246b4a6f3a126742af454aa3488105b80f5fc8faa4fc8"},"motivation":"Bimanual robot systems substantially expand manipulation capabilities, but coordinating two arms introduces additional control complexity and failure modes that are not well captured by existing benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11901","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_drugbench_dc75a792","familyId":"bmf_e6b83865905a","name":"DrugBench","oneLine":"DrugBench evaluates AI control protocols for mitigating medication-related harm using 3,671 medical conversations and FDA drug labels, covering drug interactions, contraindications, dosing constraints, and patient action restrictions. Introduces severity-based monitoring.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-10","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.20663","pdf":"https://arxiv.org/pdf/2606.20663","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.20663"},"evidence":{"snippet":"To this end, we introduce DrugBench, an AI control evaluation benchmark which combines 3,671 multi-turn medical conversations from HealthBench with drug information from official FDA labels, covering four categories of medication-related harm: drug interactions, contraindications, dosing constraints, and patient action restrictions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.20663"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DrugBench evaluates AI control protocols for mitigating medication-related harm using 3,671 medical conversations and FDA drug labels, covering drug interactions, contraindications, dosing constraints, and patient action restrictions. Introduces severity-based monitoring.","whyItMatters":"Addresses the safety-critical need to evaluate external safeguards for LLMs in medical QA, beyond simple accuracy. Useful for developers of safe medical AI systems and control protocols.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"175b15a623efd90db47501eae3eb494c23dd635f8d448b835c1d81e7434ba9e6"},"motivation":"Large Language Models have the potential to expand and improve the access to clinical information by enabling new ways of interacting with medical knowledge in natural language.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.20663","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_claw-swe-bench_ea69cc9d","familyId":"bmf_32be4dea95f8","name":"Claw-SWE-Bench","oneLine":"Claw-SWE-Bench is a multilingual SWE-bench-style benchmark and adapter protocol for comparing agent harnesses on coding tasks, with 350 GitHub issue-resolution instances across 8 languages and 43 repositories. Score is Pass@1 on patch correctness.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-10","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.12344","pdf":"https://arxiv.org/pdf/2606.12344","project":null,"code":"https://github.com/opensquilla/claw-swe-bench","data":"https://huggingface.co/datasets/TokenRhythm/Claw-SWE-Bench","hfPaper":"https://huggingface.co/papers/2606.12344"},"evidence":{"snippet":"We introduce Claw-SWE-Bench, a multilingual SWE-bench-style benchmark and adapter protocol that makes heterogeneous agent harnesses, or claws, comparable under fair settings including a fixed prompt, runtime budget, workspace contract, patch extraction procedure, and evaluator.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":71,"hfDailySubmittedAt":"2026-06-11T00:00:00.000Z","githubStars":95,"githubScope":"benchmark_repo","hfDatasetDownloads":1135,"hfDatasetLikes":6},"source":{"type":"arxiv","id":"2606.12344"},"ranking":{"90d":{"score":92,"rank":5,"coverage":1.0,"confidence":"High","datasetDownloadRank":7,"datasetRankPopulation":51}},"description":"Claw-SWE-Bench is a multilingual SWE-bench-style benchmark and adapter protocol for comparing agent harnesses on coding tasks, with 350 GitHub issue-resolution instances across 8 languages and 43 repositories. Score is Pass@1 on patch correctness.","whyItMatters":"Enables fair comparison of autonomous coding agents by treating harness and cost as first-class evaluation axes. Useful for developers and researchers building general-purpose coding agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f8bb0766fa75a120dfa40c1479579a5d65c458ccac51b1b83bcc162300dbf3fb"},"motivation":"General-purpose agents such as OpenClaw are increasingly used as autonomous tool users, but their coding ability is difficult to measure under SWE-bench: a generic agent does not by itself satisfy the clean Docker workspace, patch, and prediction contract required for scoring.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.12344","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TokenRhythm Technologies","organizationType":"company-research-lab","sourceUrl":"https://github.com/opensquilla/claw-swe-bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_worldolympiad_08793185","familyId":"bmf_9371008df044","name":"WorldOlympiad","oneLine":"WorldOlympiad is a triathlon-style benchmark for video-based world models, evaluating physical faithfulness, geometric consistency, and interaction fidelity across gaming, robotics, and general real-world scenarios. It includes 1,000 long videos and interpretable automatic metrics.","area":"Vision & 3D","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11129","pdf":"https://arxiv.org/pdf/2606.11129","project":"https://alibaba-damo-academy.github.io/WorldOlympiad/","code":"https://github.com/alibaba-damo-academy/WorldOlympiad","data":null,"hfPaper":"https://huggingface.co/papers/2606.11129"},"evidence":{"snippet":"We introduce WorldOlympiad, a benchmark for diagnosing video-based world models across physical faithfulness, geometric consistency, and interaction fidelity.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":32,"hfDailySubmittedAt":"2026-06-10T00:00:00.000Z","githubStars":55,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.11129"},"ranking":{"90d":{"score":77,"rank":33,"coverage":0.7,"confidence":"Medium"}},"description":"WorldOlympiad is a triathlon-style benchmark for video-based world models, evaluating physical faithfulness, geometric consistency, and interaction fidelity across gaming, robotics, and general real-world scenarios. It includes 1,000 long videos and interpretable automatic metrics.","whyItMatters":"Existing video generation benchmarks focus on visual quality or short-term coherence, lacking diagnostics for physical rule compliance, 3D structural consistency, and long-horizon interactive control. WorldOlympiad provides a structured protocol to identify specific failure modes in world models, informing model development and selection for embodied and interactive applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0ba2afb42eaefe4f642c77b35a3a0b41369eaddc1ffc113fb0e3c4f008b915c6"},"motivation":"We introduce WorldOlympiad, a benchmark for diagnosing video-based world models across physical faithfulness, geometric consistency, and interaction fidelity.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11129","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Alibaba DAMO Academy","organizationType":"company-research-lab","sourceUrl":"https://alibaba-damo-academy.github.io/WorldOlympiad/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_workflow-gym_72fcdcd4","familyId":"bmf_bc38808cdab5","name":"Workflow-GYM","oneLine":"Workflow-GYM evaluates AI agents on long-horizon GUI tasks in professional domains, using specialized software environments and economically valuable workflows. Tasks require end-to-end operation of graphical user interfaces, with success rates measured by task completion.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11042","pdf":"https://arxiv.org/pdf/2606.11042","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.11042"},"evidence":{"snippet":"To bridge this gap, we introduce Workflow-GYM, a benchmark for long-horizon GUI tasks centered on professional domains and specialized software environments.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":22,"hfDailySubmittedAt":"2026-06-10T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.11042"},"ranking":{"90d":{"score":55,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Workflow-GYM evaluates AI agents on long-horizon GUI tasks in professional domains, using specialized software environments and economically valuable workflows. Tasks require end-to-end operation of graphical user interfaces, with success rates measured by task completion.","whyItMatters":"Most GUI benchmarks cover simple, short-horizon tasks in general software, leaving a gap for professional, long-horizon workflows. Workflow-GYM provides a fixed protocol for measuring agent performance on such tasks, which is valuable as organizations consider deploying agents for complex professional work.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"badacf3a03dd44823e3abf9dd0292873ffea1d6474fd00c6cb1085e6cffd3063"},"motivation":"Recent years have witnessed the rapid evolution of AI agents toward handling increasingly complex, real-world tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11042","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Workflow-GYM Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.11042","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_umi-bench_1bf436a3","familyId":"bmf_b7b6ea4bab62","name":"UMI-Bench","oneLine":"UMI-Bench 1.0 is a benchmark for tabletop robotic manipulation policies using the Universal Manipulation Interface, with a unified protocol for data collection, reset, execution, and logging.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.10382","pdf":"https://arxiv.org/pdf/2606.10382","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.10382"},"evidence":{"snippet":"We present UMI-Bench 1.0, a local-first real-robot benchmark for standardized evaluation of UMI-style manipulation policies.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.10382"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"UMI-Bench 1.0 is a benchmark for tabletop robotic manipulation policies using the Universal Manipulation Interface, with a unified protocol for data collection, reset, execution, and logging.","whyItMatters":"Real-robot evaluation is crucial for assessing manipulation policies beyond curated demos; UMI-Bench aims to standardize evaluation for UMI-style policies, which could aid comparisons and reproducibility.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6079c9110175d6d6a2fb2d02525386190a09fdfd6523782622625793def40019"},"motivation":"Real-robot evaluation is essential for understanding whether learned manipulation policies can operate reliably outside curated demonstrations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.10382","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_t1-bench_01df3d35","familyId":"bmf_dbcfe3a64553","name":"T1-Bench","oneLine":"T1-Bench evaluates agentic systems through realistic customer-facing, multi-domain tasks. Scenarios involve multi-turn user-assistant interactions across 25 domains with varying difficulty, measuring structured reasoning, tool utilization, and conversational quality. Automatic evaluation is complemented by human judgments.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11070","pdf":"https://arxiv.org/pdf/2606.11070","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.11070"},"evidence":{"snippet":"To address these limitations, we introduce T1-Bench, a high-fidelity, comprehensive benchmark for evaluating agentic systems in realistic customer-facing, multi-domain environments, featuring interleaved scenarios that require structured reasoning across multi-turn user-assistant interactions and substantially increasing both compositional complexity and evaluative rigor across 25 domains of varying difficulty.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.11070"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"T1-Bench evaluates agentic systems through realistic customer-facing, multi-domain tasks. Scenarios involve multi-turn user-assistant interactions across 25 domains with varying difficulty, measuring structured reasoning, tool utilization, and conversational quality. Automatic evaluation is complemented by human judgments.","whyItMatters":"Existing agent benchmarks lack realism and domain diversity, limiting assessment of sustained multi-step reasoning. T1-Bench addresses this gap by providing interleaved, complex scenarios across multiple domains, enabling more accurate evaluation of agents' practical capabilities in real-world customer service settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6398f5c4a2cce8c21857c40b86c8bd3ca282bc018316f5e2ee279296e0b20e33"},"motivation":"Recent advances in reasoning and tool-calling capabilities of large language models (LLMs) have enabled increasingly capable agentic systems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11070","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_stage-claw_61b30974","familyId":"bmf_002eae0d56e7","name":"STAGE-Claw","oneLine":"STAGE-Claw is an automated framework for building and evaluating personal-agent tasks in state-based computing environments, with a benchmark of 40 tasks. Evaluations measure final system state correctness.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.10394","pdf":"https://arxiv.org/pdf/2606.10394","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.10394"},"evidence":{"snippet":"Given a task hint, STAGE-Claw automatically creates and validates a realistic benchmark task with its environment, task prompts, ground truth, and related verification programs.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.10394"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"STAGE-Claw is an automated framework for building and evaluating personal-agent tasks in state-based computing environments, with a benchmark of 40 tasks. Evaluations measure final system state correctness.","whyItMatters":"The framework addresses the need for scalable and realistic evaluation of personal agents, moving beyond sandboxed and static tasks to state-based verification. This supports progress in agent reliability and practical deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"54d9942e93beef802a2cd9ebc1975ee378cf0b8dc7519b187f6a0826ebbafb06"},"motivation":"Large language models are increasingly used to power personal agents for everyday applications, but evaluating these agents remains a challenge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.10394","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_socsci-repro-bench_8a1a65b5","familyId":"bmf_3424da46c0af","name":"SocSci-Repro-Bench","oneLine":"SocSci-Repro-Bench evaluates AI coding agents on reproducing social science findings from 221 tasks across four disciplines and 13 domains, using studies with known reproducibility outcomes.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11447","pdf":"https://arxiv.org/pdf/2606.11447","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.11447"},"evidence":{"snippet":"Here we introduce SocSci-Repro-Bench, a benchmark of 221 tasks spanning four disciplines and 13 substantive domains, constructed from studies whose results are either fully reproducible with available materials or demonstrably non-reproducible due to missing data, allowing us to isolate agents' reproduction capacity.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.11447"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SocSci-Repro-Bench evaluates AI coding agents on reproducing social science findings from 221 tasks across four disciplines and 13 domains, using studies with known reproducibility outcomes.","whyItMatters":"SocSci-Repro-Bench addresses the lack of systematic evaluation of AI agents' ability to execute computational workflows, providing a protocol to isolate agent performance from issues in reproduction materials and informing practical use of agents in scientific production.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cead3849ff644a663b207bd72d807853980df1f2c8c6f7f62f7d4e77ab5f3a31"},"motivation":"Recent anecdotal evidence suggests that AI coding agents can reproduce published findings when provided with original data and code; yet systematic evaluation across social sciences remains limited.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11447","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_skillresolve-bench_99845262","familyId":"bmf_df773b457612","name":"SkillResolve-Bench","oneLine":"SkillResolve-Bench 1.0 evaluates agent skill retrieval under same-capability ambiguity, pairing helpful skills with risky siblings. It includes 661 pairs, a 7,982-candidate pool, disjoint splits, and reports Recall@K and harmful sibling rate (HSR@K).","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.10388","pdf":"https://arxiv.org/pdf/2606.10388","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.10388"},"evidence":{"snippet":"We introduce SkillResolve-Bench 1.0, an auditable benchmark for this setting with 661 helpful/risky pairs, source-role and admission evidence, cue/leakage checks, query-disjoint splits, and a 7,982-candidate pool that includes 6,660 public SkillRet candidates.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.10388"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SkillResolve-Bench 1.0 evaluates agent skill retrieval under same-capability ambiguity, pairing helpful skills with risky siblings. It includes 661 pairs, a 7,982-candidate pool, disjoint splits, and reports Recall@K and harmful sibling rate (HSR@K).","whyItMatters":"Skill retrieval carries execution risk beyond relevance. This benchmark quantifies exposure to risky siblings, supporting development of retrievers that select safe representatives, reducing harmful failures in agent deployments.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a39b09a08c49f30abc0881fb8d26b02163a742f40d34c9d96279d74301e94972"},"motivation":"Agent skill libraries are becoming routable software assets: a retrieved skill can contribute instructions, scripts, resource bindings, and execution assumptions to an agent.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.10388","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_sciconbench_a454d95f","familyId":"bmf_2d99508d3db9","name":"SciConBench","oneLine":"SciConBench evaluates AI agents on open-domain scientific conclusion synthesis using 9,110 questions and expert-written conclusions from systematic reviews. The evaluation decomposes conclusions into atomic facts and measures correctness and comprehensiveness via factual precision and recall.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.11337","pdf":"https://arxiv.org/pdf/2606.11337","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.11337"},"evidence":{"snippet":"We introduce SciConBench, a large-scale live benchmark of 9.11K questions and expert-written conclusions from systematic reviews to evaluate open-domain scientific conclusion synthesis.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.11337"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SciConBench evaluates AI agents on open-domain scientific conclusion synthesis using 9,110 questions and expert-written conclusions from systematic reviews. The evaluation decomposes conclusions into atomic facts and measures correctness and comprehensiveness via factual precision and recall.","whyItMatters":"The benchmark addresses the lack of reliable evaluation for AI agents summarizing scientific evidence in health and other critical fields, providing a way to measure factual accuracy and completeness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"368e8ff0280e8b5296a4fb92a5d1ad53d87c589dc570ca33575e3dca4ccc9010"},"motivation":"Scientific AI agents increasingly retrieve evidence, reason across sources, and synthesize conclusions used in consequential decisions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11337","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_phystool-bench_e6d3d9ac","familyId":"bmf_2635f961d28a","name":"PhysTool-Bench","oneLine":"PhysTool-Bench evaluates multimodal LLMs on physical tool use through two tasks: recognizing all tools in a scene and selecting and sequencing tools for a given task, using 2,510 queries over 2,678 real-world tools.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Tool use"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.10803","pdf":"https://arxiv.org/pdf/2606.10803","project":null,"code":"https://github.com/ModalityDance/PhysTool-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.10803"},"evidence":{"snippet":"To address this gap, we introduce PhysTool-Bench, the first physical tool-use benchmark designed to evaluate MLLMs' ability to comprehend real-world scenarios, identify physical tools, and plan their use.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.10803"},"ranking":{"90d":{"score":31,"rank":312,"coverage":0.7,"confidence":"Medium"}},"description":"PhysTool-Bench evaluates multimodal LLMs on physical tool use through two tasks: recognizing all tools in a scene and selecting and sequencing tools for a given task, using 2,510 queries over 2,678 real-world tools.","whyItMatters":"Physical tool use is underexplored in MLLMs; this benchmark isolates recognition and planning deficits, supporting progress in embodied AI and practical human-robot collaboration.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a6a97b05aa973e34994b7af031a29d2daf8f7f3edeaaf56c4d8b92a3b7adf350"},"motivation":"Multimodal Large Language Models (MLLMs) excel at utilizing digital APIs and increasingly serve as the \"brain\" of embodied AI, instructing robots to interact with the physical world.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.10803","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ModalityDance","organizationType":"academic-lab","sourceUrl":"https://github.com/ModalityDance/PhysTool-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Agents","Tool Calling"],"domainScope":"general"},{"id":"bm_phantombench_848c0022","familyId":"bmf_b3df0dc70b12","name":"PhantomBench","oneLine":"PhantomBench evaluates language models' ability to abstain from answering about non-existent entities. It comprises over 60,000 non-existent terms derived from real concepts across domains, and provides a pipeline for generating further instances.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.11105","pdf":"https://arxiv.org/pdf/2606.11105","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.11105"},"evidence":{"snippet":"We introduce PhantomBench, the first large-scale benchmark of its kind, comprising more than 60K non-existent terms and entities derived from real concepts across diverse domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.11105"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"PhantomBench evaluates language models' ability to abstain from answering about non-existent entities. It comprises over 60,000 non-existent terms derived from real concepts across domains, and provides a pipeline for generating further instances.","whyItMatters":"Addresses the evaluation gap in assessing models' calibration of knowledge boundaries, offering a practical tool for detecting hallucination tendencies in high-stakes applications and studying behavior on rare concepts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2a4e7d3c08cd8f27206db425569fa7bfe03ef54c20ad42a71dfe683da21bace1"},"motivation":"Hallucinations, where language models (LMs) generate factually ungrounded responses, pose serious risks, as users tend to blindly rely on them.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11105","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_p3d-bench_31251204","familyId":"bmf_15e7f389af7b","name":"P3D-Bench","oneLine":"P3D-Bench evaluates multimodal large language models on parametric 3D generation from text, image, and assembly specifications, scoring executability, geometric fidelity, topology, text-grounded constraints, multiview semantic alignment, and part-level structure.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Geometric reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11152","pdf":"https://arxiv.org/pdf/2606.11152","project":"https://spatiaos.github.io/projects/P3D-Bench","code":"https://github.com/SpatiaOS/P3D-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.11152"},"evidence":{"snippet":"We introduce P3D-Bench, a benchmark for parametric 3D generation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":5,"hfDailySubmittedAt":"2026-06-15T00:00:00.000Z","githubStars":48,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.11152"},"ranking":{"90d":{"score":72,"rank":61,"coverage":0.7,"confidence":"Medium"}},"description":"P3D-Bench evaluates multimodal large language models on parametric 3D generation from text, image, and assembly specifications, scoring executability, geometric fidelity, topology, text-grounded constraints, multiview semantic alignment, and part-level structure.","whyItMatters":"Existing benchmarks rarely evaluate 3D modeling through code, which requires geometric precision and assembly consistency, not just runnable code. P3D-Bench provides a unified protocol to assess structural understanding and precise geometry, which is critical for models generating parametric 3D programs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a04a4e956a5aa5a993864106b3ff28929e0d91be761661e01f2ae597aac799d2"},"motivation":"Multimodal large language models can write code to produce complex programs as well as use programs to do 3D modeling, which opens up a new avenue for 3D generation powered by their priors, world knowledge and reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11152","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SpatiaOS","organizationType":"academic-lab","sourceUrl":"https://github.com/SpatiaOS/P3D-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_oncotraj_0ff8775a","familyId":"bmf_bdc2d0e2d2ef","name":"OncoTraj","oneLine":"OncoTraj is a public benchmark for predicting acquired resistance to first-line osimertinib in EGFR-mutant non-small-cell lung cancer. It provides a harmonized dataset of 813 patients from three real-world sources, with locked splits, an evaluation harness, and six baselines, defining three tasks: 12-month progression classification, time-to-progression regression, and resistance mechanism classification.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11144","pdf":"https://arxiv.org/pdf/2606.11144","project":null,"code":"https://github.com/span-ai-labs/oncotraj","data":"https://huggingface.co/datasets/span-ai-labs/oncotraj-v1","hfPaper":"https://huggingface.co/papers/2606.11144"},"evidence":{"snippet":"We introduce OncoTraj, a public benchmark of 813 EGFR-mutant NSCLC patients receiving first-line osimertinib, harmonized from three real-world clinical-genomic sources: MSK-CHORD (672 patients), AACR Project GENIE BPC NSCLC (34 patients), and the FLAURA molecular-resistance supplement (107 patients).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":43,"hfDatasetLikes":1},"source":{"type":"arxiv","id":"2606.11144"},"ranking":{"90d":{"score":18,"rank":395,"coverage":0.85,"confidence":"High","datasetDownloadRank":45,"datasetRankPopulation":51}},"description":"OncoTraj is a public benchmark for predicting acquired resistance to first-line osimertinib in EGFR-mutant non-small-cell lung cancer. It provides a harmonized dataset of 813 patients from three real-world sources, with locked splits, an evaluation harness, and six baselines, defining three tasks: 12-month progression classification, time-to-progression regression, and resistance mechanism classification.","whyItMatters":"OncoTraj fills the gap of a standardized, public evaluation for longitudinal resistance prediction, offering reproducible baseline results and leakage-audited splits. It provides a practical means to compare models and highlights the limitation of single-timepoint features, guiding future data collection and algorithm development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"45e7b3d746c923009561febbf9ece9a74aebae17d8fadd6f94d4d054cf38b6ec"},"motivation":"Resistance to first-line osimertinib in EGFR-mutant non-small-cell lung cancer (NSCLC) is the canonical example of predictable clonal evolution under therapeutic pressure, yet no public benchmark exists for training or evaluating computational models on the corresponding longitudinal patient trajectories.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11144","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Span AI Labs","organizationType":"company-research-lab","sourceUrl":"https://github.com/span-ai-labs/oncotraj","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_mpc-patch-bench_574ee054","familyId":"bmf_0c668e75b30f","name":"MPC-Patch-Bench","oneLine":"MPC-Patch-Bench evaluates LLM-based code repair on repository-level Secure Multi-Party Computation (MPC) software. It provides 205 verified instances with Fail-to-Pass/Pass-to-Pass tests and a verifier that checks cryptographic safety and numerical fidelity via differential testing and static analysis.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.CR"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11416","pdf":"https://arxiv.org/pdf/2606.11416","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.11416"},"evidence":{"snippet":"We introduce MPC-Patch-Bench, a repository-level benchmark organised around two frameworks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.11416"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MPC-Patch-Bench evaluates LLM-based code repair on repository-level Secure Multi-Party Computation (MPC) software. It provides 205 verified instances with Fail-to-Pass/Pass-to-Pass tests and a verifier that checks cryptographic safety and numerical fidelity via differential testing and static analysis.","whyItMatters":"Existing benchmarks lack MPC-aware evaluation for repository-level code repair. MPC-Patch-Bench addresses security and numerical-fidelity gaps, offering a repeatable protocol for assessing LLM agents on real-world MPC tasks, with verification rejected up to 40% of functionally passing patches.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5f0c20089eb1b773a816e838d7ef44624ec30c955fb68286621f83a0b5ae8754"},"motivation":"Repository-level benchmarks for evaluating Large Language Model (LLM) code repair on Secure Multi-Party Computation (MPC) software do not yet exist, and directly transplanting general-purpose benchmarks such as SWE-bench fails on three structural fronts: (i) MPC repositories are dominated by generic Python infrastructure rather than cryptographic logic; (ii) high-value MPC fixes lack the standardized tests rigid ext…","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11416","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_lakeqa_b77b7993","familyId":"bmf_dfc10241a16b","name":"LakeQA","oneLine":"LakeQA evaluates search-centric question answering over a 9.5 TB data lake of Wikipedia and government data. Tasks require multi-hop reasoning across heterogeneous structured and unstructured sources, with expert-annotated answers.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.10460","pdf":"https://arxiv.org/pdf/2606.10460","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.10460"},"evidence":{"snippet":"To this end, we introduce LakeQA, a comprehensive benchmark for search-centric question answering over data lakes that jointly emphasizes searching and reasoning capabilities.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.10460"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LakeQA evaluates search-centric question answering over a 9.5 TB data lake of Wikipedia and government data. Tasks require multi-hop reasoning across heterogeneous structured and unstructured sources, with expert-annotated answers.","whyItMatters":"Existing QA benchmarks provide explicit evidence or trivial retrieval, missing the challenge of locating and composing evidence in large-scale data lakes. LakeQA fills this gap, supporting development and assessment of agents that can search and reason over massive heterogeneous data.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b3684db45f5e76dcb55cae665a8c38bf62640d81127db03fd9d426e64b9d8c86"},"motivation":"Recent large language models (LLMs) have shown rapid progress in reading-based question answering (QA), where evidence is explicitly provided or can be trivially retrieved.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.10460","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_janus_33d6bb69","familyId":"bmf_850ca8e0ec3f","name":"Janus","oneLine":"JANUS is a benchmark with 160 scenarios across 8 domains, each providing a fixed pool of favorable and adverse facts and paired neutral and goal-directed prompts, to evaluate fact-grounded, goal-conditioned pragmatic distortion in LLM outputs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.10852","pdf":"https://arxiv.org/pdf/2606.10852","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.10852"},"evidence":{"snippet":"We introduce JANUS, a benchmark for measuring goal-conditioned pragmatic distortion in fact-grounded LLM outputs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.10852"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"JANUS is a benchmark with 160 scenarios across 8 domains, each providing a fixed pool of favorable and adverse facts and paired neutral and goal-directed prompts, to evaluate fact-grounded, goal-conditioned pragmatic distortion in LLM outputs.","whyItMatters":"Existing benchmarks primarily detect direct deception, missing subtler misleading communication that stays factually accurate. JANUS measures whether LLMs distort net impressions when incentivized, offering a more practical assessment of risks in real-world applications where selective presentation of facts can mislead stakeholders.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c2ad0b2a215ac65f5dc2350a20137a6c20035787766987448697827a7a6a842c"},"motivation":"LLM deception is often evaluated through direct markers such as fabricated claims, explicit lies, or strategic concealment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.10852","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"JANUS Benchmark Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.10852","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_combench_b27be297","familyId":"bmf_29e41fd248d6","name":"ComBench","oneLine":"ComBench evaluates large language models on 100 human-annotated Olympiad-level combinatorics problems, split into 50 analysis-centric and 50 construction-centric tasks. Scoring combines rubric-guided proof grading with deterministic verification of construction outputs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.10479","pdf":"https://arxiv.org/pdf/2606.10479","project":"https://simplified-reasoning.github.io/ComBench/docs/","code":"https://github.com/Simplified-Reasoning/ComBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.10479"},"evidence":{"snippet":"We introduce ComBench, an Olympiad-level combinatorics benchmark for evaluating and diagnosing the combinatorial reasoning capabilities of large language models.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":20,"hfDailySubmittedAt":"2026-06-11T00:00:00.000Z","githubStars":14,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.10479"},"ranking":{"90d":{"score":66,"rank":95,"coverage":0.7,"confidence":"Medium"}},"description":"ComBench evaluates large language models on 100 human-annotated Olympiad-level combinatorics problems, split into 50 analysis-centric and 50 construction-centric tasks. Scoring combines rubric-guided proof grading with deterministic verification of construction outputs.","whyItMatters":"ComBench fills a gap in evaluating creative and rigorous combinatorial reasoning at the Olympiad level. It provides separate scores for proof quality and construction validity, helping diagnose where models diverge in these capabilities, and enables fine-grained comparison of frontier models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b0e3f91093bc15da60e71ed07dcd7dcac15cba99db7a2fa710aecffbd7e374ff"},"motivation":"Combinatorics is central to Olympiad-level mathematical problem solving, requiring deep discrete reasoning, creative constructions, and rigorous structural insight.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.10479","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Simplified Reasoning","organizationType":"academic-lab","sourceUrl":"https://github.com/Simplified-Reasoning/ComBench","role":"benchmark-publisher"}],"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_ciaware-bench_e29a610f","familyId":"bmf_78a7e9aa9db3","name":"CIAware-Bench","oneLine":"CIAware-Bench measures control intervention awareness in language models through four task domains: essay writing, BigCodeBench, Bash Arena, and SHADE-Arena. Models are tested on distinguishing their own trajectories from those modified by a control protocol, with variations in watermarking, side-task presence, and protocol.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11063","pdf":"https://arxiv.org/pdf/2606.11063","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.11063"},"evidence":{"snippet":"We introduce \\textbf{CIAware-Bench}, a benchmark for measuring \\textbf{c}ontrol \\textbf{i}ntervention (CI) awareness across frontier models.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.11063"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CIAware-Bench measures control intervention awareness in language models through four task domains: essay writing, BigCodeBench, Bash Arena, and SHADE-Arena. Models are tested on distinguishing their own trajectories from those modified by a control protocol, with variations in watermarking, side-task presence, and protocol.","whyItMatters":"This benchmark addresses the evaluation gap of determining whether models can detect modifications to their trajectories, which is critical for AI control protocols. It provides practical value by informing the design of interventions that are harder for models to detect, thereby improving the robustness of AI oversight systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"69e83acb4b5b3c483e323308f788839786750feff0a7b9062cfa8af73f86a2db"},"motivation":"AI control protocols oversee untrusted models by monitoring their actions and modifying potentially unsafe steps, often using a trusted model.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11063","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_abc-bench_fecd3eb4","familyId":"bmf_5fb363ebd5d3","name":"ABC-Bench","oneLine":"ABC-Bench evaluates LLM agents on biosecurity-relevant tasks including liquid handling robot code generation, DNA fragment design, and DNA synthesis screening evasion, with wet-lab validation.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences","Robotics & Autonomous Systems"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech","Robotics"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-09","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.11150","pdf":"https://arxiv.org/pdf/2606.11150","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.11150"},"evidence":{"snippet":"To address this, we introduce the Agentic Bio-Capabilities Benchmark (ABC-Bench), a suite of tasks to measure agentic biosecurity-relevant capabilities.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.11150"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ABC-Bench evaluates LLM agents on biosecurity-relevant tasks including liquid handling robot code generation, DNA fragment design, and DNA synthesis screening evasion, with wet-lab validation.","whyItMatters":"Measures agentic AI capabilities relevant to biosecurity, offering a standardized protocol to assess dual-use risks and inform safeguards in biological research contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"46382a337c2a4c2787994e7a9a1c935baafe0b8dcfffb05409ef80c2b771a938"},"motivation":"Large language models (LLMs) are rapidly acquiring capabilities relevant to biological research, from literature synthesis to interpretation of experimental data.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.11150","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"cross-domain"},{"id":"bm_weavebench_27cb863c","familyId":"bmf_88fdd16b4aa1","name":"WeaveBench","oneLine":"WeaveBench evaluates computer-use agents on 114 long-horizon tasks across 8 real-world work domains. Each task interleaves GUI interaction with command-line and code operations in a single trajectory, with scoring based on a trajectory-aware judge that detects fabricated evidence.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09426","pdf":"https://arxiv.org/pdf/2606.09426","project":null,"code":"https://github.com/weavebench/WeaveBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.09426"},"evidence":{"snippet":"Thus, we introduce WeaveBench, a long-horizon hybrid-interface benchmark with 114 tasks across 8 real-world work domains, grounded in real user requests and publicly verifiable artifacts.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":107,"hfDailySubmittedAt":"2026-06-12T00:00:00.000Z","githubStars":159,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09426"},"ranking":{"90d":{"score":83,"rank":11,"coverage":0.7,"confidence":"Medium"}},"description":"WeaveBench evaluates computer-use agents on 114 long-horizon tasks across 8 real-world work domains. Each task interleaves GUI interaction with command-line and code operations in a single trajectory, with scoring based on a trajectory-aware judge that detects fabricated evidence.","whyItMatters":"Existing benchmarks often evaluate interfaces separately, leaving hybrid orchestration under-tested. WeaveBench fills this gap by requiring agents to combine GUI and CLI/code in realistic tasks, and exposes that outcome-only grading overestimates performance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"af8ec21e146fe5a6035ccd9fe211bba62061a42025bd35f8560232cc962ab2ae"},"motivation":"Computer-use agents (CUAs) increasingly operate in runtimes that combine visual desktop control, command-line execution, code editing, browsers, and external tools.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09426","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"WeaveBench team","organizationType":"academic-lab","sourceUrl":"https://github.com/weavebench/WeaveBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_uxbench_9ba76902","familyId":"bmf_8d180072bdf4","name":"UXBench","oneLine":"UXBench evaluates user experience in AI assistants through three tasks: UX Judge (binary classification of response quality), UX Eval (response generation), and UX Recovery (repairing failed interactions). The dataset contains 7,400 test instances from 70K+ real interaction logs, covering 8 scenarios and 83 domains. Scoring uses accuracy for Judge and GRM-rated quality for generation tasks.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09570","pdf":"https://arxiv.org/pdf/2606.09570","project":null,"code":"https://github.com/mengze-hong/UXBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.09570"},"evidence":{"snippet":"We present UXBench, the first user-centric benchmark grounded in real user feedback signals for evaluating preference alignment and dialogue generation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":4,"hfDailySubmittedAt":null,"githubStars":15,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09570"},"ranking":{"90d":{"score":63,"rank":113,"coverage":0.7,"confidence":"Medium"}},"description":"UXBench evaluates user experience in AI assistants through three tasks: UX Judge (binary classification of response quality), UX Eval (response generation), and UX Recovery (repairing failed interactions). The dataset contains 7,400 test instances from 70K+ real interaction logs, covering 8 scenarios and 83 domains. Scoring uses accuracy for Judge and GRM-rated quality for generation tasks.","whyItMatters":"UXBench addresses the gap in evaluating AI assistants beyond raw capability, focusing on user-perceived utility and preference alignment. It provides a structured way to measure how well models understand and improve user experience, offering practical value for developing assistants that better satisfy real users.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"189775d557918107eabb50b66a7917c5b1edc6e09eed26da6dc9b5754ef0339c"},"motivation":"As AI assistants serve millions of users daily, evaluating user experience (UX) beyond general model capability has become increasingly important.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09570","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_trl-bench_b244de7b","familyId":"bmf_722ae64b1c7b","name":"TRL-Bench","oneLine":"TRL-Bench evaluates tabular encoders at the representation level by exporting row-, column-, or table-level embeddings through each encoder's supported wrapper and probing them with shared lightweight heads across three suites: TRL-CTbench (column/table), TRL-Rbench (row), and TRL-DLTE (compositional Data-Lake Table Enrichment) covering 16 tasks and 20 models.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09323","pdf":"https://arxiv.org/pdf/2606.09323","project":null,"code":"https://github.com/LOGO-CUHKSZ/TRL-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.09323"},"evidence":{"snippet":"We introduce TRL-Bench, a multi-granular tabular representation learning (TRL) benchmark that standardizes cross-paradigm representation-level evaluation: each encoder exports row-, column-, or table embeddings through its supported wrapper, and shared lightweight heads probe them across three suites: TRL-CTbench (column/table), TRL-Rbench (row), and TRL-DLTE (compositional Data-Lake Table Enrichment spanning all three granularities).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":53,"hfDailySubmittedAt":"2026-06-11T00:00:00.000Z","githubStars":10,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09323"},"ranking":{"90d":{"score":65,"rank":100,"coverage":0.7,"confidence":"Medium"}},"description":"TRL-Bench evaluates tabular encoders at the representation level by exporting row-, column-, or table-level embeddings through each encoder's supported wrapper and probing them with shared lightweight heads across three suites: TRL-CTbench (column/table), TRL-Rbench (row), and TRL-DLTE (compositional Data-Lake Table Enrichment) covering 16 tasks and 20 models.","whyItMatters":"Traditional end-to-end pipelines obscure the comparative quality of tabular encoders from different training paradigms. TRL-Bench provides a standardized protocol to isolate representation-level capability, enabling model selection based on task-specific strengths rather than a single aggregate score.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"75d17630044b0d81b27f75e8919bf9f8009ba719fd915f9545b8ae5a650f9be4"},"motivation":"Tabular encoders are usually evaluated inside task-specific end-to-end pipelines, so models from different training paradigms are difficult to compare directly even when they operate on similar tabular signals.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09323","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LOGO Lab, CUHK-Shenzhen","organizationType":"academic-lab","sourceUrl":"https://github.com/LOGO-CUHKSZ/TRL-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_theorembench_8bd203fd","familyId":"bmf_e8410a03abbd","name":"TheoremBench","oneLine":"Evaluates LLMs on theorem proving in Lean4 using classical theorems, with two versions: main and premised. Includes metrics for theorem-level coverage and token efficiency to assess partial progress and proof structure.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09450","pdf":"https://arxiv.org/pdf/2606.09450","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.09450"},"evidence":{"snippet":"We introduce TheoremBench, a Lean4 benchmark designed to evaluate theorem provers beyond contest settings.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09450"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates LLMs on theorem proving in Lean4 using classical theorems, with two versions: main and premised. Includes metrics for theorem-level coverage and token efficiency to assess partial progress and proof structure.","whyItMatters":"Provides a more realistic evaluation of provers beyond contest problems, revealing biases toward easy subtheorems and inefficient proof strategies. Supports finer-grained analysis of formal reasoning capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"d903f06021d84a12e5ccc9bb1a83178e23d4f6730d1725d9b4118f5c72bd16ad"},"motivation":"LLMs have recently achieved strong results on formal proving benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09450","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TheoremBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.09450","role":"benchmark-publisher"}],"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_tabverse_d67e32ec","familyId":"bmf_4624dae44f51","name":"TABVERSE","oneLine":"TABVERSE is a controlled multimodal benchmark that aligns the same table content across HTML, Markdown, LaTeX, and rendered images, with question category and difficulty tags. It evaluates LLMs and VLMs on question answering, structural understanding, and structure reconstruction.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09578","pdf":"https://arxiv.org/pdf/2606.09578","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.09578"},"evidence":{"snippet":"We introduce TABVERSE, a controlled multimodal table benchmark that aligns the same table content across multiple structural formats and rendered images, with question category and difficulty tags.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09578"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"TABVERSE is a controlled multimodal benchmark that aligns the same table content across HTML, Markdown, LaTeX, and rendered images, with question category and difficulty tags. It evaluates LLMs and VLMs on question answering, structural understanding, and structure reconstruction.","whyItMatters":"Existing table benchmarks conflate content, format, and modality, obscuring the impact of representation choice. TABVERSE enables isolation of representation effects, providing practical guidance for selecting robust table formats for downstream applications and evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"44564f038262799f85352a80ee39a3c6ef3727ff6cada747f12f71fedd6e7ea5"},"motivation":"Large Language Models (LLMs) and Vision-Language Models (VLMs) are increasingly evaluated on table reasoning tasks, but the role of table representation remains under-explored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09578","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_st-bench_eeee9e2f","familyId":"bmf_c12a5ba8d0fc","name":"ST-Bench","oneLine":"ST-Bench is a verification benchmark for evaluating certified robustness of spatio-temporal neural networks on autonomous driving (Udacity) and activity recognition (UCF-101) tasks, using spatio-temporal perturbation constraints.","area":"Safety & Trustworthiness","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Automotive"],"capabilities":["Robustness"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.09746","pdf":"https://arxiv.org/pdf/2606.09746","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.09746"},"evidence":{"snippet":"To spur further progress in this field, we propose ST-Bench, a verification benchmark for autonomous driving and activity recognition, to systematically evaluate verifiable robustness.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09746"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ST-Bench is a verification benchmark for evaluating certified robustness of spatio-temporal neural networks on autonomous driving (Udacity) and activity recognition (UCF-101) tasks, using spatio-temporal perturbation constraints.","whyItMatters":"Existing robustness verification methods rely on overly conservative assumptions or are computationally prohibitive. ST-Bench provides a realistic, constrained perturbation model for video inputs, enabling tighter approximations and more meaningful robustness comparisons for safety-critical applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"85a0aa69d53bb7351576bac24f6aca41b9443a1ff5d32b6f96ce3155c4c63d0c"},"motivation":"With AI increasingly deployed in safety-critical systems, providing formal robustness guarantees for the underlying models is essential.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"9th International Symposium on AI Verification (SAIV 2026)","evidence":"Accepted at the 9th International Symposium on AI Verification (SAIV 2026)","evidenceUrl":"https://arxiv.org/abs/2606.09746","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"9th International Symposium on AI Verification (SAIV 2026)","reviewStatus":"accepted","decisionRaw":"Accepted at the 9th International Symposium on AI Verification (SAIV 2026)","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.09746","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at the 9th International Symposium on AI Verification (SAIV 2026)","level":"author-claim"}]}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_spatialworld_4f21023e","familyId":"bmf_b45842946201","name":"SpatialWorld","oneLine":"SpatialWorld evaluates multimodal agents on interactive spatial reasoning in 760 real-world tasks across eight simulation backends. Agents operate under vision-only partial observability, using a unified text-based action interface. Performance is measured via terminal-state verifiers for task success rate and step efficiency.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09669","pdf":"https://arxiv.org/pdf/2606.09669","project":null,"code":"https://github.com/Hongcheng-Gao/SpatialWorld","data":null,"hfPaper":"https://huggingface.co/papers/2606.09669"},"evidence":{"snippet":"We introduce SpatialWorld, a unified benchmark designed specifically for evaluating the interactive spatial understanding of multimodal agents in complex real-world tasks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":48,"hfDailySubmittedAt":"2026-06-09T00:00:00.000Z","githubStars":54,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09669"},"ranking":{"90d":{"score":78,"rank":29,"coverage":0.7,"confidence":"Medium"}},"description":"SpatialWorld evaluates multimodal agents on interactive spatial reasoning in 760 real-world tasks across eight simulation backends. Agents operate under vision-only partial observability, using a unified text-based action interface. Performance is measured via terminal-state verifiers for task success rate and step efficiency.","whyItMatters":"Existing benchmarks rely on passive VQA or simulator-specific pipelines, failing to assess interactive spatial understanding. SpatialWorld provides a unified, simulator-agnostic protocol with human-validated evaluation, revealing that current models achieve low success rates, highlighting gaps in active exploration and long-horizon planning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0d7ddb4d37209814fd6ecfb0cd6b5bbf2aa8c346230263f46848ee81b8fe5600"},"motivation":"Spatial reasoning is a foundational capability for multimodal large language models (MLLMs) to perceive and operate within the physical world.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09669","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SpatialWorld Team","organizationType":"academic-lab","sourceUrl":"https://github.com/Hongcheng-Gao/SpatialWorld","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_rtl-bench_4e1de99d","familyId":"bmf_d6fea90599a9","name":"RTL-Bench","oneLine":"RTL-BenchLS evaluates LLMs on RTL design generation and reasoning, containing over 10,000 formally verified Verilog designs. Tasks include specification-to-RTL generation, round-trip reasoning, masked-content reasoning, and repository-issue reasoning. All tasks are verified via formal equivalence checking without manual testbenches.","area":"Language & Knowledge","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Semiconductors"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08976","pdf":"https://arxiv.org/pdf/2606.08976","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.08976"},"evidence":{"snippet":"We introduce RTL-BenchLS, a large-scale benchmark addressing both limitations above.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08976"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RTL-BenchLS evaluates LLMs on RTL design generation and reasoning, containing over 10,000 formally verified Verilog designs. Tasks include specification-to-RTL generation, round-trip reasoning, masked-content reasoning, and repository-issue reasoning. All tasks are verified via formal equivalence checking without manual testbenches.","whyItMatters":"Existing RTL benchmarks are small and saturate with frontier models. RTL-BenchLS provides a large-scale, challenging benchmark with self-supervised tasks, enabling tracking of progress on complex hardware design reasoning and generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"177e66c8502318178b5e2467d723a9ae68b8c370faee281d179fb09c165b7248"},"motivation":"LLM-based RTL generation and reasoning is a promising direction for hardware design automation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08976","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_popsicle_93cd1103","familyId":"bmf_39a63441d299","name":"POPSICLE","oneLine":"POPSICLE benchmarks cryoET segmentation and macromolecular localization using data from the CryoET Data Portal. It spans eukaryotic and prokaryotic systems, purified and in situ samples, and covers dense voxel-wise segmentation and sparse localization tasks. Built on the CryoET Data Portal, it can expand with new data.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["eess.IV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.10255","pdf":"https://arxiv.org/pdf/2606.10255","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.10255"},"evidence":{"snippet":"Here, we present POPSICLE, a benchmark suite for cryoET segmentation and macromolecular localization built from the CryoET Data Portal - an open, ML-ready repository of tomographic data, metadata, and annotations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.10255"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"POPSICLE benchmarks cryoET segmentation and macromolecular localization using data from the CryoET Data Portal. It spans eukaryotic and prokaryotic systems, purified and in situ samples, and covers dense voxel-wise segmentation and sparse localization tasks. Built on the CryoET Data Portal, it can expand with new data.","whyItMatters":"CryoET lacks standardized, well-annotated benchmarks, limiting robust comparison across methods. POPSICLE provides an open, extensible foundation from a living repository, with baseline experiments showing task-dependent model rankings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"507b64bb6c98fe6e010d730383e7ab831f7b5231c7681db69459252ece1bf400"},"motivation":"Cryo-electron tomography (cryoET) has emerged as a powerful tool in structural and cellular biology by enabling direct visualization of macromolecular structures within intact cells, thereby linking molecular architecture to cellular organization in a native context.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.10255","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_omnigamearena_6afe515a","familyId":"bmf_bdedbd5c5172","name":"OmniGameArena","oneLine":"OmniGameArena evaluates VLM game agents across twelve Unreal Engine 5 games spanning Solo, PvP, and Coop play. It provides unified action interfaces and two evaluation clocks (PDQ for decision quality, LCRT for real-time latency). The Improvement Dynamics Curve (IDC) measures agent improvement through reflection rounds.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.09826","pdf":"https://arxiv.org/pdf/2606.09826","project":null,"code":"https://github.com/mxlin043/OmniGameArena","data":null,"hfPaper":"https://huggingface.co/papers/2606.09826"},"evidence":{"snippet":"We address these gaps with OmniGameArena, a real-time benchmark of twelve newly built Unreal Engine 5 games spanning Solo (7), PvP (3), and Coop (2) with unified action interfaces, and the Improvement Dynamics Curve (IDC), an agentic-reflection harness in which a tool-using reflector LLM autonomously refines a bounded skill prompt across multiple rounds.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":19,"hfDailySubmittedAt":"2026-06-09T00:00:00.000Z","githubStars":41,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09826"},"ranking":{"90d":{"score":74,"rank":54,"coverage":0.7,"confidence":"Medium"}},"description":"OmniGameArena evaluates VLM game agents across twelve Unreal Engine 5 games spanning Solo, PvP, and Coop play. It provides unified action interfaces and two evaluation clocks (PDQ for decision quality, LCRT for real-time latency). The Improvement Dynamics Curve (IDC) measures agent improvement through reflection rounds.","whyItMatters":"Existing game benchmarks report single scores and lack unified protocols for heterogeneous agents. OmniGameArena offers a real-time benchmark with multiple observables, including improvement dynamics, facilitating comparative evaluation of different agent classes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a05d9d2d817c0f5f2799299c7ebb66c137ba23d2cefb0bb95254e2e1a85a137e"},"motivation":"Vision-language model (VLM) agents are increasingly deployed in interactive game environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09826","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"OmniGameArena Team","organizationType":"academic-lab","sourceUrl":"https://github.com/mxlin043/OmniGameArena","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_normbench_a8bf7215","familyId":"bmf_8a7f98f06ad7","name":"NormBench","oneLine":"NormBench evaluates defeasible scope parsing in legal texts using Span-Grounded Deontic Trees, with 2,290 provisions across multiple languages. It focuses on identifying clause overrides and includes whole-tree fidelity metrics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08932","pdf":"https://arxiv.org/pdf/2606.08932","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.08932"},"evidence":{"snippet":"To diagnose and mitigate SSO, we introduce NormBench, a benchmark of 2,290 provisions spanning Chinese (laws and local policies), English (U.S.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08932"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"NormBench evaluates defeasible scope parsing in legal texts using Span-Grounded Deontic Trees, with 2,290 provisions across multiple languages. It focuses on identifying clause overrides and includes whole-tree fidelity metrics.","whyItMatters":"Silent Scope Omission is a critical failure in rule-following agents. NormBench provides a diagnostic benchmark to identify structural omissions and improve statutory understanding in LLMs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"56dbc27679f4534c96d8a9c38161aeb241021744fead820c3614890513af863d"},"motivation":"Rule-following agents tasked with executing policies and regulations often fail via Silent Scope Omission (SSO): a model applies a general rule but silently drops nested exceptions or counter-exceptions, producing outputs that appear compliant yet break on important edge cases.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08932","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_lexrubric_7a24242e","familyId":"bmf_8419445f9469","name":"LexRubric","oneLine":"LexRubric evaluates open-ended legal tasks in Chinese, with 649 instances from legal consultation and judicial examination. It includes 12,337 expert-written atomic scoring criteria under a six-dimensional framework, enabling fine-grained diagnostic evaluation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09389","pdf":"https://arxiv.org/pdf/2606.09389","project":null,"code":"https://github.com/foggpoy/LexRubric","data":null,"hfPaper":"https://huggingface.co/papers/2606.09389"},"evidence":{"snippet":"We introduce LexRubric, a rubric-based benchmark for evaluating open-ended Chinese legal tasks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09389"},"ranking":{"90d":{"score":35,"rank":306,"coverage":0.7,"confidence":"Medium"}},"description":"LexRubric evaluates open-ended legal tasks in Chinese, with 649 instances from legal consultation and judicial examination. It includes 12,337 expert-written atomic scoring criteria under a six-dimensional framework, enabling fine-grained diagnostic evaluation.","whyItMatters":"Open-ended legal responses require fine-grained evaluation beyond exact matching. LexRubric provides rubric-based diagnostic assessment, showing distinct capability profiles across models and highlighting challenges in open-ended legal questions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"56b0ecb68d99a353289d7c154ada1d195d40459f80b74ca556b558b82401028e"},"motivation":"As large language models (LLMs) are increasingly applied to real-world legal tasks, evaluating the reliability of their open-ended legal responses has become essential.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09389","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LexRubric Team","organizationType":"academic-lab","sourceUrl":"https://github.com/foggpoy/LexRubric","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_iosworld_171f5010","familyId":"bmf_5df7fedfa2e7","name":"iOSWorld","oneLine":"iOSWorld evaluates phone agents on interactive tasks within a native iOS simulator. It includes 26 custom iOS apps with connected personal data, 133 tasks across single-app, multi-app, and memory/personalization categories. Agents are evaluated with rubric-based scoring.","area":"Agents & Tool Use","applicationDomains":["Consumer & Productivity"],"primaryDomain":"Consumer & Productivity","industrySectors":["Consumer Technology"],"capabilities":["Computer use","Cross-app planning","Memory & personalization"],"topics":["Agents","Mobile","Personalization"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09764","pdf":"https://arxiv.org/pdf/2606.09764","project":"https://iosworld.io/","code":"https://github.com/ljang0/iOSWorld","data":null,"hfPaper":"https://huggingface.co/papers/2606.09764"},"evidence":{"snippet":"We introduce iOSWorld, the first interactive native iOS simulator benchmark built around a persistent user identity spanning 26 newly built iOS apps.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-reviewed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":4,"hfDailySubmittedAt":"2026-06-18T00:00:00.000Z","githubStars":15,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09764"},"ranking":{"90d":{"score":63,"rank":114,"coverage":0.7,"confidence":"Medium"}},"description":"iOSWorld evaluates phone agents on interactive tasks within a native iOS simulator. It includes 26 custom iOS apps with connected personal data, 133 tasks across single-app, multi-app, and memory/personalization categories. Agents are evaluated with rubric-based scoring.","whyItMatters":"Existing mobile benchmarks lack personalization and interactive evaluation. iOSWorld provides a benchmark with persistent user identity and multi-app tasks, showing significant gaps in multi-app and memory-based performance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a8045efb73c464679ea35312b31b67d30645c825b22ef2dfac3fa1eb6a8ccc91"},"motivation":"A useful phone agent needs to be personally intelligent.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"source-reviewed","reviewedAt":"2026-08-19","sources":["https://arxiv.org/abs/2606.09764","https://iosworld.io/","https://github.com/ljang0/iOSWorld"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09764","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"iOSWorld Team","organizationType":"academic-lab","sourceUrl":"https://github.com/ljang0/iOSWorld","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_imug-bench_282ebfce","familyId":"bmf_92e790a6df66","name":"IMUG-Bench","oneLine":"IMUG-Bench evaluates unified multimodal models on multi-turn interleaved image-text understanding and generation. It includes 3,113 samples and 12,034 interaction turns across Static Spatial, Temporal Causal, and Hybrid classes, with dynamic understanding questions.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09169","pdf":"https://arxiv.org/pdf/2606.09169","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.09169"},"evidence":{"snippet":"To bridge this gap, we propose IMUG-Bench, a comprehensive benchmark for multi-turn interleaved image-text dialogue of UMMs that jointly evaluates their understanding and generation capabilities.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09169"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"IMUG-Bench evaluates unified multimodal models on multi-turn interleaved image-text understanding and generation. It includes 3,113 samples and 12,034 interaction turns across Static Spatial, Temporal Causal, and Hybrid classes, with dynamic understanding questions.","whyItMatters":"Existing benchmarks fail to evaluate multi-turn interleaved interactions and expose bias. IMUG-Bench provides a comprehensive evaluation revealing capability boundaries and failure modes, and explores test-time scaling strategies.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ce091803bc0a1f379bbd008d46778b66cba1c3da185d636365d11f760279076c"},"motivation":"In recent years, unified multimodal models (UMMs) have emerged to support both understanding and generation within a single framework.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09169","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_h2hmem_6775c266","familyId":"bmf_5903d1e9207d","name":"H2HMem","oneLine":"H2HMem is a benchmark for evaluating memory capabilities of agents in human-human multimodal interactions, covering dyadic and multi-party conversations with tasks in memory recall, reasoning, and application.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09461","pdf":"https://arxiv.org/pdf/2606.09461","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.09461"},"evidence":{"snippet":"To address this gap, we introduce H2HMem, a Human-to-Human Multimodal Memory Benchmark for evaluating memory capabilities in complex human-human interactions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09461"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"H2HMem is a benchmark for evaluating memory capabilities of agents in human-human multimodal interactions, covering dyadic and multi-party conversations with tasks in memory recall, reasoning, and application.","whyItMatters":"Existing memory benchmarks focus on single-user text interactions; H2HMem addresses the need for evaluating agents in complex multimodal human-human settings with asynchronous and conflicting information from multiple participants.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1dd0b93d91fc092466bc89b8b73897273985ddbd246c507c7d855c0c25cf004d"},"motivation":"Large language model agents are increasingly deployed in human-human interaction settings, such as meeting assistants and clinical documentation systems, where they must observe conversations and retain information for downstream queries.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09461","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_erbench_e22588c9","familyId":"bmf_58b25273f95f","name":"ERBench","oneLine":"ERBench is an evaluation framework for equation discovery algorithms, assessing recovery of groundtruth formulas under varying dimensionality, sampling size, distribution, and domain.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09276","pdf":"https://arxiv.org/pdf/2606.09276","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.09276"},"evidence":{"snippet":"To fill this gap, we introduce the Equation Recovery Benchmark (ERBench), a new evaluation framework designed to rigorously assess algorithms explicitly targeting the task of equation discovery.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09276"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"ERBench is an evaluation framework for equation discovery algorithms, assessing recovery of groundtruth formulas under varying dimensionality, sampling size, distribution, and domain.","whyItMatters":"Existing symbolic regression benchmarks have few public groundtruth formulas and limited robustness testing; ERBench aims to provide a more rigorous evaluation for algorithm practitioners.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"68d4418438db88d0dc070d2f8e2582d8e611fbc1c41ee78630a211bf222c206e"},"motivation":"Equation discovery aims to automate the discovery of scientific models in the form of mathematical equations from data.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09276","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_egotactile_e93c424a","familyId":"bmf_ffce4b8dd699","name":"EgoTactile","oneLine":"EgoTactile is a dataset pairing egocentric video with full-hand pressure supervision for everyday objects, including a bare-hand transfer subset.","area":"Vision & 3D","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09243","pdf":"https://arxiv.org/pdf/2606.09243","project":"https://egotactile.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.09243"},"evidence":{"snippet":"Therefore, we introduce EgoTactile, a benchmark pairing egocentric video with full-hand pressure supervision for diverse everyday objects, incorporating a bare-hand transfer subset to enable generalization to natural scenarios.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09243"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"EgoTactile is a dataset pairing egocentric video with full-hand pressure supervision for everyday objects, including a bare-hand transfer subset.","whyItMatters":"The dataset supports research in estimating grasp pressure from egocentric video, which is relevant for VR and robotic manipulation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a2c31baf1b64c5e1fec7f2cc17285fdc77d831cc43a87b15a55cb585f15d8fba"},"motivation":"Estimating full-hand grasp pressure from egocentric video is critical for immersive VR and robotic manipulation, yet dense tactile sensing often relies on intrusive hardware.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML2026 spotlight","evidence":"Accepted to ICML2026 spotlight","evidenceUrl":"https://arxiv.org/abs/2606.09243","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML2026 spotlight","reviewStatus":"accepted","decisionRaw":"Accepted to ICML2026 spotlight","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.09243","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to ICML2026 spotlight","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_ego-mc-bench_91c3532a","familyId":"bmf_4df235e7cb24","name":"Ego-MC-Bench","oneLine":"Ego-MC-Bench is a benchmark for evaluating reactive step-by-step task guidance in cooking scenarios, focusing on timely interventions when mistakes occur.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09547","pdf":"https://arxiv.org/pdf/2606.09547","project":"https://apratimbh.github.io/livecookv2/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.09547"},"evidence":{"snippet":"To evaluate this crucial capability, we introduce Ego-MC-Bench (Mistake Corrections), a benchmark for evaluating reactive, step-by-step task guidance in realistic cooking scenarios.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09547"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Ego-MC-Bench is a benchmark for evaluating reactive step-by-step task guidance in cooking scenarios, focusing on timely interventions when mistakes occur.","whyItMatters":"The benchmark addresses the capability of video LLMs to intervene proactively during task execution, which is crucial for practical guidance assistants.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"efe3fea054f05e12cc57d5c8a88984974ac3199414ebe5a629dea5eda6950954"},"motivation":"Learning everyday skills, like cooking a dish, relies increasingly on instructional media such as online videos.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09547","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_distract-bench_3873c48a","familyId":"bmf_74ca191db947","name":"Distract-Bench","oneLine":"Distract-Bench evaluates robustness of vision-language models to semantic visual distractions, which are meaningful but task-irrelevant cues that preserve the ground-truth answer.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Robustness"],"topics":["Multimodal","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08894","pdf":"https://arxiv.org/pdf/2606.08894","project":null,"code":"https://github.com/Yizheng-Sun/Distract-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.08894"},"evidence":{"snippet":"To address this gap, we introduce \\textbf{Distract-Bench}, a benchmark for evaluating VLM robustness to \\textbf{semantic visual distractions}, defined as meaningful but task-irrelevant visual cues added to inputs while preserving the ground-truth answer.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":10,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08894"},"ranking":{"90d":{"score":58,"rank":142,"coverage":0.55,"confidence":"Low"}},"description":"Distract-Bench evaluates robustness of vision-language models to semantic visual distractions, which are meaningful but task-irrelevant cues that preserve the ground-truth answer.","whyItMatters":"It exposes a distinct failure mode where models perceive evidence correctly but reason from distracting cues, shifting robustness evaluation from perceptual degradation to distraction handling.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3a746734efbdf9199a373fc70cc1f55de7066e6c16c76f7437758e552168964d"},"motivation":"Reasoning Vision-Language Models (VLMs) achieve strong performance on complex multimodal tasks, but reliable real-world application requires handling visual inputs that are messier than clean, curated benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08894","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Yizheng-Sun","organizationType":"community","sourceUrl":"https://github.com/Yizheng-Sun/Distract-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_contextshift_bf97ab71","familyId":"bmf_91aae91c3f82","name":"ContextShift","oneLine":"ContextShift is a controlled benchmark that manipulates object-context relationships in COCO images to isolate context as an independent variable for object detection evaluation.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09495","pdf":"https://arxiv.org/pdf/2606.09495","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.09495"},"evidence":{"snippet":"We introduce ContextShift, a controlled benchmark that systematically manipulates object--context relationships while preserving object appearance.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09495"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ContextShift is a controlled benchmark that manipulates object-context relationships in COCO images to isolate context as an independent variable for object detection evaluation.","whyItMatters":"It reveals that standard aggregate metrics like AP can mask substantial recall loss and changes in prediction dynamics under context variation, aiding in understanding detector robustness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"311d196a3ae562287f829cd97f4a4b3e0fe207e85f2ed8750375192431ab34b9"},"motivation":"Modern object detectors achieve strong performance on standard benchmarks, yet their robustness to contextual variation remains insufficiently understood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09495","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_bensyc_50720d15","familyId":"bmf_7197a91c5412","name":"BenSyc","oneLine":"BenSyc is a benchmark for studying conversational sycophancy in Bengali social contexts, with binary labels and a five-level taxonomy from invalidation to escalation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-08","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.10061","pdf":"https://arxiv.org/pdf/2606.10061","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.10061"},"evidence":{"snippet":"We introduce BenSyc, the first benchmark for studying conversational sycophancy in Bengali social contexts.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":"2026-06-10T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.10061"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"BenSyc is a benchmark for studying conversational sycophancy in Bengali social contexts, with binary labels and a five-level taxonomy from invalidation to escalation.","whyItMatters":"It fills a gap in sycophancy research by focusing on culturally grounded conversational alignment in Bengali, important for socially aligned conversational AI.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3876527e6a234d1ab3875db4479c7c08bd5527343dec92d674037da17c7f8e0f"},"motivation":"Large language models (LLMs) increasingly participate in emotionally sensitive social conversations, where responses may shift from balanced support toward excessive validation or escalatory alignment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.10061","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_zipbench_2610de18","familyId":"bmf_71f839953810","name":"ZIPBench","oneLine":"ZIPBench is a zero-shot personalization benchmark with 1.5K users, graph-mined personas, and 40K generated images for evaluating text-to-image personalization.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-07","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.08841","pdf":"https://arxiv.org/pdf/2606.08841","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.08841"},"evidence":{"snippet":"We introduce ZIPBench, the first zero-shot personalization benchmark with 1.5K users, graph-mined personas, and 40K generated images.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08841"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"ZIPBench is a zero-shot personalization benchmark with 1.5K users, graph-mined personas, and 40K generated images for evaluating text-to-image personalization.","whyItMatters":"It addresses the need for evaluating personalization without user-specific data, assessing alignment with individual aesthetic preferences.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"52cf7f930ff0f25beae5c29e882e9e91d63d983a51672d0c940dc713beb51f67"},"motivation":"Text-to-image diffusion models are increasingly deployed in open-ended creative contexts, yet their outputs remain impersonal, optimized for aggregate aesthetics rather than individual taste.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08841","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_pipe-cypher_c078ae44","familyId":"bmf_2fdb7077d9b5","name":"PIPE-Cypher","oneLine":"PIPE-Cypher is a pipeline that generates NL-to-Cypher benchmarks from live property graphs, producing executable query pairs with validation, diversity controls, and local LLM judges.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08481","pdf":"https://arxiv.org/pdf/2606.08481","project":null,"code":"https://github.com/suraj-ranganath/PIPE-Cypher","data":null,"hfPaper":"https://huggingface.co/papers/2606.08481"},"evidence":{"snippet":"We present PIPE-Cypher, a local benchmark-generation pipeline that turns a live property graph and optional seed queries from customer questions, analyst logs, or agent tool calls into balanced NL-to-Cypher benchmarks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":"2026-06-09T00:00:00.000Z","githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08481"},"ranking":{"90d":{"score":25,"rank":392,"coverage":0.7,"confidence":"Medium"}},"description":"PIPE-Cypher is a pipeline that generates NL-to-Cypher benchmarks from live property graphs, producing executable query pairs with validation, diversity controls, and local LLM judges.","whyItMatters":"Enterprise graph schemas and query patterns are unique and evolve, making static benchmarks obsolete. PIPE-Cypher enables repeatable, graph-specific evaluation of text-to-Cypher systems, supporting deployment-relevant performance assessment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e302ac4fea949d587350ca60e3b062e2c020656546bab6ccf270966c4a4cd33b"},"motivation":"Enterprise property graphs vary widely in schema structure, internal terminology, domain assumptions, governance constraints, and user interaction patterns.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08481","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"PIPE-Cypher Developers","organizationType":"community","sourceUrl":"https://github.com/suraj-ranganath/PIPE-Cypher","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_perspectivegap_bc1804c7","familyId":"bmf_56adfd1bcc0a","name":"PerspectiveGap","oneLine":"PerspectiveGap is a benchmark with 110 scenarios for evaluating multi-agent orchestration prompting, using distractor-mixed tasks and topologies from the authors' practice.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08878","pdf":"https://arxiv.org/pdf/2606.08878","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.08878"},"evidence":{"snippet":"To measure this, we introduce PerspectiveGap, a benchmark for evaluating LLMs' ability to compose orchestration prompts for multi-agent systems.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08878"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"PerspectiveGap is a benchmark with 110 scenarios for evaluating multi-agent orchestration prompting, using distractor-mixed tasks and topologies from the authors' practice.","whyItMatters":"Multi-agent orchestration prompting is an emerging capability; the benchmark aims to measure it systematically, but the evaluation protocol and artifacts are not publicly accessible.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c015c9f93dddb6a67e3713ed223dd8036fc969e6f1d9b701a49c627692ac36db"},"motivation":"Real-world LLM applications are moving beyond single-agent workflows toward orchestrated multi-agent systems, yet current models still struggle to determine what each sub-agent needs to know.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08878","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_omnicap-if_98c29629","familyId":"bmf_3e95cd9aa687","name":"OmniCap-IF","oneLine":"OmniCap-IF evaluates instruction following in omni-modal video captioning with 50 constraint types, 1,920 samples, and checklist-based scoring for format and content correctness.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08572","pdf":"https://arxiv.org/pdf/2606.08572","project":null,"code":"https://github.com/NJU-LINK/omnicap-if","data":null,"hfPaper":"https://huggingface.co/papers/2606.08572"},"evidence":{"snippet":"To bridge this gap, we introduce OmniCap-IF, the first comprehensive benchmark specifically designed to evaluate instruction-following capabilities in omni-modal captioning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":14,"hfDailySubmittedAt":"2026-06-09T00:00:00.000Z","githubStars":11,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08572"},"ranking":{"90d":{"score":64,"rank":110,"coverage":0.7,"confidence":"Medium"}},"description":"OmniCap-IF evaluates instruction following in omni-modal video captioning with 50 constraint types, 1,920 samples, and checklist-based scoring for format and content correctness.","whyItMatters":"Existing benchmarks miss the interplay of audio-visual and user constraints; OmniCap-IF provides a fine-grained evaluation to expose format-content tradeoffs and drive improvements.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8c8ffb534a25992679278a1e0d6143005bf114ca3eed4f0946c1a9b4ef0d8164"},"motivation":"While Omni-modal Large Language Models (OLLMs) have demonstrated impressive capabilities in jointly processing audio and visual streams, their ability to strictly adhere to complex, multi-faceted user instructions remains largely unexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08572","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"NJU-LINK","organizationType":"academic-lab","sourceUrl":"https://github.com/NJU-LINK/omnicap-if","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_lilybench_6840fe07","familyId":"bmf_91b640010f26","name":"LilyBench","oneLine":"LilyBench evaluates symbolic music generation and understanding using LilyPond, with a 200-prompt generation suite and ten understanding tasks, scored via compile rate, descriptor similarity, and FMD.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SD"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08722","pdf":"https://arxiv.org/pdf/2606.08722","project":null,"code":"https://github.com/CSCPadova/lilybench","data":null,"hfPaper":"https://huggingface.co/papers/2606.08722"},"evidence":{"snippet":"We introduce LilyBench, a LilyPond-based benchmark that jointly evaluates symbolic music generation and music understanding on the same family of open-weight LLMs.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08722"},"ranking":{"90d":{"score":25,"rank":393,"coverage":0.7,"confidence":"Medium"}},"description":"LilyBench evaluates symbolic music generation and understanding using LilyPond, with a 200-prompt generation suite and ten understanding tasks, scored via compile rate, descriptor similarity, and FMD.","whyItMatters":"Symbolic music evaluation is fragmented; LilyBench provides a joint benchmark with public datasets and code, enabling reproducible comparisons and metric triangulation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"75b31e09fe9c1b5c0864b81279efdeb6db326d7cca4fb175816fdbab6d07f515"},"motivation":"Symbolic music evaluation for large language models remains fragmented across representations, datasets, and metrics.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"Ital-IA 2026","evidence":"Accepted at Ital-IA 2026","evidenceUrl":"https://arxiv.org/abs/2606.08722","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"Ital-IA 2026","reviewStatus":"accepted","decisionRaw":"Accepted at Ital-IA 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.08722","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at Ital-IA 2026","level":"author-claim"}]}],"publishers":[{"name":"CSCPadova","organizationType":"academic-lab","sourceUrl":"https://github.com/CSCPadova/lilybench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_covebench_94152ad0","familyId":"bmf_9e4ba492c102","name":"CoVEBench","oneLine":"CoVEBench evaluates compositional video editing with 416 source videos, 626 multi-point instructions, and 9,990 checklist items, using MLLM judges and objective metrics.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-07","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08415","pdf":"https://arxiv.org/pdf/2606.08415","project":null,"code":"https://github.com/NJU-LINK/CoVEBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.08415"},"evidence":{"snippet":"To address this gap, we introduce CoVEBench, a compositional video editing benchmark comprising 416 curated source videos, 626 multi-point editing instructions, and 9,990 fine-grained checklist items.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":52,"hfDailySubmittedAt":"2026-06-09T00:00:00.000Z","githubStars":19,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08415"},"ranking":{"90d":{"score":69,"rank":79,"coverage":0.7,"confidence":"Medium"}},"description":"CoVEBench evaluates compositional video editing with 416 source videos, 626 multi-point instructions, and 9,990 checklist items, using MLLM judges and objective metrics.","whyItMatters":"Realistic video editing requires handling multiple coupled edits; CoVEBench provides a diagnostic testbed to reveal failures in complex instruction compliance and preservation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"24827a5a6257e03926cf514f6bd049e46bdc32156cb1b47d4448e6f561c11219"},"motivation":"While recent text-guided video editing models excel at elementary tasks (e.g., style transfer, object insertion), real-world user requests are highly compositional.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08415","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"NJU-LINK Team","organizationType":"academic-lab","sourceUrl":"https://github.com/NJU-LINK/CoVEBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_videoweaver_b86f354a","familyId":"bmf_339e1bf0a622","name":"VideoWeaver","oneLine":"VideoWeaver is an agent harness and benchmark for long video generation, with 16 task categories and 285 cases, evaluating agents via evidence-grounded judge on process and output.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08091","pdf":"https://arxiv.org/pdf/2606.08091","project":null,"code":"https://github.com/JianhuiWei7/VideoWeaver","data":null,"hfPaper":"https://huggingface.co/papers/2606.08091"},"evidence":{"snippet":"We introduce VideoWeaver, an agent harness and benchmark that evaluates and evolves skills for long video generation, where an agent turns a single instruction into a long video by composing foundation skills into its own workflow rather than following a predefined pipeline.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":31,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08091"},"ranking":{"90d":{"score":67,"rank":90,"coverage":0.55,"confidence":"Low"}},"description":"VideoWeaver is an agent harness and benchmark for long video generation, with 16 task categories and 285 cases, evaluating agents via evidence-grounded judge on process and output.","whyItMatters":"General-purpose agents are underevaluated on long-horizon multimodal tasks; VideoWeaver offers a reproducible benchmark to assess and evolve agent skills for video generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fdbb1580023e736af8479d8f50e230407c22dc1e90a1dded8e41bb195f62d981"},"motivation":"Recent agent frameworks such as Claude Code, Codex, and OpenClaw are strong at tool use and orchestration, but whether they can handle long video generation, a long-horizon multimodal task, remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08091","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"JianhuiWei7","organizationType":"community","sourceUrl":"https://github.com/JianhuiWei7/VideoWeaver","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_uniql_126b9bdb","familyId":"bmf_989990ea6746","name":"UniQL","oneLine":"UniQL is a human-verified benchmark for cross-dialect text-to-SQL, aligning 1,534 questions with executable SQL across 16 dialects, with dialect-specific evaluation tracks.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08018","pdf":"https://arxiv.org/pdf/2606.08018","project":null,"code":"https://github.com/JerryGao818/UniQL","data":null,"hfPaper":"https://huggingface.co/papers/2606.08018"},"evidence":{"snippet":"We introduce UniQL, a human-verified benchmark for cross-dialect text-to-SQL evaluation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08018"},"ranking":{"90d":{"score":39,"rank":296,"coverage":0.55,"confidence":"Low"}},"description":"UniQL is a human-verified benchmark for cross-dialect text-to-SQL, aligning 1,534 questions with executable SQL across 16 dialects, with dialect-specific evaluation tracks.","whyItMatters":"Most text-to-SQL benchmarks are SQLite-only, but real systems use varied dialects; UniQL enables controlled evaluation of dialect generalization, highlighting transfer gaps.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3b4b4796ee2a186ddd8f0433b283c03b0f55191b2ee53590fee6ce7860ac6e9e"},"motivation":"Existing text-to-SQL benchmarks are largely centered on SQLite, making it difficult to evaluate whether models can generalize across heterogeneous SQL dialects.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08018","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"JerryGao818","organizationType":"community","sourceUrl":"https://github.com/JerryGao818/UniQL","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_simple_0e42de19","familyId":"bmf_a7a39b72f297","name":"SIMPLE","oneLine":"SIMPLE is a simulation testbed for humanoid loco-manipulation with 60 tasks, 50 scenes, and over 1,000 objects, integrating data generation pipelines and benchmarking policies.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08278","pdf":"https://arxiv.org/pdf/2606.08278","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.08278"},"evidence":{"snippet":"To this end, we present SIMPLE, a unified simulation testbed for humanoid policy learning and evaluation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08278"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SIMPLE is a simulation testbed for humanoid loco-manipulation with 60 tasks, 50 scenes, and over 1,000 objects, integrating data generation pipelines and benchmarking policies.","whyItMatters":"Real-world humanoid evaluation is expensive; SIMPLE aims to provide a reproducible simulation benchmark, but public access to environments and data is not yet confirmed.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d25bde345149b06e4b4424f0ad30fcc3229dc65a3afa65bdbf407477681885a9"},"motivation":"Humanoid foundation models are advancing faster than we can evaluate them.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08278","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_idp-bench_f6df4252","familyId":"bmf_c03dcd02a45c","name":"IDP-Bench","oneLine":"IDP-Bench evaluates large language models on interdependent privacy scenarios, covering recognition of co-ownership, identification of contextual integrity parameters, and judgments of sharing appropriateness.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-06","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.09908","pdf":"https://arxiv.org/pdf/2606.09908","project":null,"code":"https://github.com/tisl-lab/Interdependent_Privacy_Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.09908"},"evidence":{"snippet":"We address this gap by introducing \\textbf{IDP-Bench}: the first LLM benchmark for IDP scenarios, grounded in the Contextual Integrity (CI) framework.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09908"},"ranking":{"90d":{"score":29,"rank":377,"coverage":0.55,"confidence":"Low"}},"description":"IDP-Bench evaluates large language models on interdependent privacy scenarios, covering recognition of co-ownership, identification of contextual integrity parameters, and judgments of sharing appropriateness.","whyItMatters":"Interdependent privacy is a critical yet underexplored risk when LLMs act as personal assistants; a reusable benchmark enables systematic comparison and improvement of models' privacy reasoning in shared-data contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6de12f0be02d1f103a75c882aa71c5e42385c223a4214625b8c5f6d8eb1f0d8c"},"motivation":"Large language models (LLMs) are becoming widely deployed as personal AI assistants with access to sensitive user data, making privacy a major challenge for their design and evaluation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09908","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TISL Lab","organizationType":"academic-lab","sourceUrl":"https://github.com/tisl-lab/Interdependent_Privacy_Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_globeaudio_abb3482b","familyId":"bmf_575d54451a6e","name":"GlobeAudio","oneLine":"GlobeAudio evaluates audio-language models on naturalistic audio understanding across six languages. It includes 5,637 multiple-choice questions with naturally occurring audio, testing auditory reasoning and cultural interpretation.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08194","pdf":"https://arxiv.org/pdf/2606.08194","project":null,"code":null,"data":"https://huggingface.co/datasets/iNLP-Lab/GlobeAudio","hfPaper":"https://huggingface.co/papers/2606.08194"},"evidence":{"snippet":"To bridge this gap, we propose GlobeAudio, a multilingual and multicultural benchmark designed to evaluate naturalistic audio understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":4,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":135,"hfDatasetLikes":2},"source":{"type":"arxiv","id":"2606.08194"},"ranking":{"90d":{"score":51,"rank":191,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":28,"datasetRankPopulation":51}},"description":"GlobeAudio evaluates audio-language models on naturalistic audio understanding across six languages. It includes 5,637 multiple-choice questions with naturally occurring audio, testing auditory reasoning and cultural interpretation.","whyItMatters":"Existing benchmarks lack linguistic and cultural authenticity and acoustic realism. GlobeAudio addresses this gap for comparison of models under real-world conditions, highlighting performance differences, especially for open-source models and low-resource languages.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"890c02f52ae757e4b09c10cf947ceb93b2f83c1d8dc44b5400724c346c227a3f"},"motivation":"Large Audio-Language Models (LALMs) integrate audio perception and language understanding within a unified framework, enabling a wide range of real-world applications.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08194","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"iNLP-Lab","organizationType":"academic-lab","sourceUrl":"https://huggingface.co/datasets/iNLP-Lab/GlobeAudio","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_gischolarbench_a6158b85","familyId":"bmf_373e6eeda3f2","name":"GIScholarBench","oneLine":"GIScholarBench evaluates LLM overconfidence in GIS research across three tasks: metadata retrieval, literature linking, and research direction generation, using 10,865 papers from 25 GIScience journals.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.IR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.08036","pdf":"https://arxiv.org/pdf/2606.08036","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.08036"},"evidence":{"snippet":"To examine this issue, we introduce GIScholarBench, a benchmark built from 10,865 papers published in 25 core GIScience journals between 2020 and 2025.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.08036"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GIScholarBench evaluates LLM overconfidence in GIS research across three tasks: metadata retrieval, literature linking, and research direction generation, using 10,865 papers from 25 GIScience journals.","whyItMatters":"It addresses the need for benchmarks that assess factual accuracy and overconfidence in scholarly AI applications, providing a basis for evaluating LLM reliability in research workflows.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1db69c2d5bb190d6d8d9398e0a81157fa3e6add49640ed2634fca67489141dcc"},"motivation":"Large language models (LLMs) are increasingly used in academic research workflows, but scholarly tasks require high factual precision and therefore expose a key weakness: overconfidence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.08036","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_chronophybench_c3f107b4","familyId":"bmf_b04565b83dc6","name":"ChronoPhyBench","oneLine":"ChronoPhyBench evaluates multimodal LLMs on chronological physical dynamics reasoning via next-state prediction and VQA, using video frames and captions for single-image selection and multi-frame sorting.","area":"Safety & Trustworthiness","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-06","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07962","pdf":"https://arxiv.org/pdf/2606.07962","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07962"},"evidence":{"snippet":"Motivated by this, and to rigorously mitigate language modality bias and shortcuts, we propose a novel multimodal Chrono}logical Physical Dynamics Reasoning Benchmark ChronoPhyBench, which unifies next state prediction with Visual Question Answering (VQA) paradigms by conditioning on historical video context and textual captions to enforce models to deduce subsequent physical states through both single image selection and the inherently more complex task of multiple frame chronological sorting.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07962"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ChronoPhyBench evaluates multimodal LLMs on chronological physical dynamics reasoning via next-state prediction and VQA, using video frames and captions for single-image selection and multi-frame sorting.","whyItMatters":"Addresses the gap of distinguishing true multimodal reasoning from language-prior exploitation, providing a robust framework to measure physical reasoning and hallucination rates in models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5bd1375b6ae9121f7d0eef7ce0cf80d9d83dde07d58b319f355faf630e17b75a"},"motivation":"Recent advancements in Multimodal Large Language Models (MLLMs) have demonstrated remarkable proficiency in open-world reasoning and understanding.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07962","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ChronoPhyBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.07962","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_visualflip_556bf533","familyId":"bmf_eb3ea10a5081","name":"VisualFLIP","oneLine":"VisualFLIP evaluates multimodal LLMs on visual reasoning with 1,374 images in paired perturbation tasks. Each pair has a fixed question but minimally changed visual evidence so the answer flips. Scoring uses pair accuracy and Collapse Rate to test evidence dependence in capabilities like cardinality, attribute, spatial, and logic.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07872","pdf":"https://arxiv.org/pdf/2606.07872","project":"https://didizhu-judy.github.io/VisualFLIP/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07872"},"evidence":{"snippet":"We introduce VisualFLIP, a paired benchmark with 1,374 images arranged as same-question perturbation pairs across cardinality, attribute, spatial, and logic tasks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07872"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"VisualFLIP evaluates multimodal LLMs on visual reasoning with 1,374 images in paired perturbation tasks. Each pair has a fixed question but minimally changed visual evidence so the answer flips. Scoring uses pair accuracy and Collapse Rate to test evidence dependence in capabilities like cardinality, attribute, spatial, and logic.","whyItMatters":"Accuracy alone can hide flawed reasoning. VisualFLIP exposes whether models truly rely on task-critical visual changes, distinguishing robust grounding from guesswork. This helps select models for high-stakes visual tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f885a1b3364ce9ea25680e46db3dd46f70e2747bab2b7ef4b08044cddcf7878b"},"motivation":"When a multimodal large language model answers a visual reasoning question correctly, is the prediction actually supported by the task-critical visual evidence?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07872","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"VisualFLIP team","organizationType":"academic-lab","sourceUrl":"https://didizhu-judy.github.io/VisualFLIP/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_urdummlu_7d4881d1","familyId":"bmf_60682f36bc94","name":"UrduMMLU","oneLine":"UrduMMLU evaluates Urdu language understanding through 26,431 multiple-choice questions across 26 subjects and five domains, sourced from native educational materials. Accuracy under zero-shot and few-shot prompting protocols serves as the primary scoring metric.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07167","pdf":"https://arxiv.org/pdf/2606.07167","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07167"},"evidence":{"snippet":"We introduce UrduMMLU, a benchmark of 26,431 Urdu MCQs across 26 subjects and five domains, collected from native Urdu MCQ banks and public examination PDFs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07167"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"UrduMMLU evaluates Urdu language understanding through 26,431 multiple-choice questions across 26 subjects and five domains, sourced from native educational materials. Accuracy under zero-shot and few-shot prompting protocols serves as the primary scoring metric.","whyItMatters":"Urdu, spoken by over 230 million people, lacks broad MMLU-style evaluation from native sources. UrduMMLU addresses this gap by testing models on region-specific knowledge, revealing uneven performance across subjects and providing a public benchmark for comparing LLMs on Urdu understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"36a90b37cdc68070a3b03bfc002d161b1ac854b5dd7d2d81a60d0358fd947750"},"motivation":"Meaningful multilingual evaluation must test models in the target language and educational context.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07167","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_swe-marathon_2e8e782b","familyId":"bmf_fbbe5c4b7d4a","name":"SWE-Marathon","oneLine":"SWE-Marathon evaluates AI agents on 20 ultra-long-horizon software engineering tasks, each with a unique executable environment, a human-written reference solution, and a multi-layer verification suite. Tasks average 27.2M tokens per logged agent attempt, requiring sustained progress over hours and millions of tokens.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Code"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07682","pdf":"https://arxiv.org/pdf/2606.07682","project":"https://swe-marathon.org/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07682"},"evidence":{"snippet":"We introduce SWE-Marathon, a benchmark of 20 long-horizon tasks spanning software engineering and adjacent technical domains.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07682"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SWE-Marathon evaluates AI agents on 20 ultra-long-horizon software engineering tasks, each with a unique executable environment, a human-written reference solution, and a multi-layer verification suite. Tasks average 27.2M tokens per logged agent attempt, requiring sustained progress over hours and millions of tokens.","whyItMatters":"Existing agent benchmarks focus on short tasks, limiting measurement of planning, long-context understanding, and memory. SWE-Marathon addresses the gap by providing a longer-horizon evaluation that exposes practical limitations in agent autonomy and highlights failure modes like reward hacking, informing development of more robust agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4a8516fea7e8c588b9b3aa7d8155f8d86deddf6e4bbc0af148466d0996221ea2"},"motivation":"AI agents are increasingly expected to complete long-horizon workflows that require sustained progress over hours, millions of tokens, and complex environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07682","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"swe-marathon.org","organizationType":"community","sourceUrl":"https://swe-marathon.org/","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_swe-explore_ece5d09b","familyId":"bmf_1b67c9f91522","name":"SWE-Explore","oneLine":"SWE-Explore evaluates repository exploration by coding agents: given a repository and an issue, an explorer returns a ranked list of relevant code regions under a fixed line budget. Ground truth is line-level, derived from successful repair trajectories. Coverage, ranking, and context-efficiency metrics are scored.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07297","pdf":"https://arxiv.org/pdf/2606.07297","project":null,"code":"https://github.com/Qiushao-E/SWE-Explore-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.07297"},"evidence":{"snippet":"In this paper, we introduce SWE-Explore, a benchmark that isolates the evaluation of repository exploration, a critical capability of coding agents.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":123,"hfDailySubmittedAt":"2026-06-09T00:00:00.000Z","githubStars":42,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07297"},"ranking":{"90d":{"score":77,"rank":36,"coverage":0.7,"confidence":"Medium"}},"description":"SWE-Explore evaluates repository exploration by coding agents: given a repository and an issue, an explorer returns a ranked list of relevant code regions under a fixed line budget. Ground truth is line-level, derived from successful repair trajectories. Coverage, ranking, and context-efficiency metrics are scored.","whyItMatters":"Existing repository-level benchmarks treat coding tasks as a single resolved/unresolved outcome, obscuring whether an agent locates the right context. SWE-Explore isolates exploration quality, enabling targeted evaluation of retrieval and localization capabilities that precede patch generation. Its metrics track downstream repair behavior, offering practical value for improving agent design.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1648e19fc75788d56e640247218b1b7764bbe77aae0bea5d3e5143dfb337fcc2"},"motivation":"Repository-level coding benchmarks such as SWE-bench have driven a rapid surge in the capabilities of coding agents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07297","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_sopribench_12eedd2a","familyId":"bmf_4c1de46427bd","name":"SopriBench","oneLine":"SopriBench evaluates user-level privacy leakage from social media posts across text, images, and metadata in 50 synthetic profiles with 1,569 images, scoring via the Privacy Exposure Score (PES) that weights value granularity by contextual sensitivity.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CR"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06784","pdf":"https://arxiv.org/pdf/2606.06784","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.06784"},"evidence":{"snippet":"To address these gaps, we propose SopriBench, a synthetic benchmark guided by leakage patterns abstracted from a private reference corpus of Rednote and Instagram accounts, covering 50 user profiles and 1,569 images with attributes, contextual sensitivity, granularity, leakage type, inference difficulty, and supporting evidence.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06784"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SopriBench evaluates user-level privacy leakage from social media posts across text, images, and metadata in 50 synthetic profiles with 1,569 images, scoring via the Privacy Exposure Score (PES) that weights value granularity by contextual sensitivity.","whyItMatters":"This benchmark addresses the gap in evaluating cumulative cross-post privacy leakage, providing a metric that captures exposure severity rather than binary accuracy, aiding in assessing real-world privacy risks from social media data.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b263b18199ab3aad17385941fe9d79d17046bd29c40867517d2eddb4e4064005"},"motivation":"Public social media posts can reveal private information through weak cues scattered across text, images, or metadata.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.06784","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_remedi_11832147","familyId":"bmf_500f28f269c5","name":"REMEDI","oneLine":"REMEDI is a benchmark for machine unlearning in multi-label clinical disease inference, built on MIMIC-III, covering diverse forget sets and tasks with utility and unlearning metrics.","area":"Safety & Trustworthiness","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Safety"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07141","pdf":"https://arxiv.org/pdf/2606.07141","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07141"},"evidence":{"snippet":"To this end, we introduce REMEDI, an extensive benchmark for machine unlearning tailored to multi-label and multiclass clinical disease inference, where label correlations, longitudinal structure, and safety constraints make unlearning particularly challenging.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07141"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"REMEDI is a benchmark for machine unlearning in multi-label clinical disease inference, built on MIMIC-III, covering diverse forget sets and tasks with utility and unlearning metrics.","whyItMatters":"It provides a realistic medical-domain evaluation for machine unlearning methods, addressing the lack of benchmarks that reflect real-world patient data and multi-label scenarios, which is crucial for privacy-preserving AI in healthcare.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d8f91238afed47dda5124258f3eae926aec71f2b1d37abeec169b54bc3aa8c07"},"motivation":"Language models trained for clinical disease inference are trained on patient data, which may include sensitive and private information, and data owners may request the removal of their data from a trained model due to privacy or copyright concerns.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07141","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_realdocbench_6da01042","familyId":"bmf_f035a1fb06db","name":"RealDocBench","oneLine":"RealDocBench evaluates field-level QA and layout understanding on real regulated documents, with 1,356 field-level questions over 581 documents and 1,500 annotated page images, scored on per-field accuracy and adjacency-aware layout metrics.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.07401","pdf":"https://arxiv.org/pdf/2606.07401","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07401"},"evidence":{"snippet":"We introduce RealDocBench, a two-track benchmark built from real regulated documents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07401"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RealDocBench evaluates field-level QA and layout understanding on real regulated documents, with 1,356 field-level questions over 581 documents and 1,500 annotated page images, scored on per-field accuracy and adjacency-aware layout metrics.","whyItMatters":"It addresses the gap in document parsing evaluation by focusing on real-world regulated documents and specific field-level needs, enabling cost-aware comparisons of commercial and open-source systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a0e2f39a9d4bf85b28d3b984ddccb01bba575c8d2a795f922bae59dab2c23e51"},"motivation":"Document parsing systems are increasingly deployed in high-stakes, regulated workflows such as mortgage underwriting, financial reporting, supply-chain logistics, and clinical records.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07401","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_phun-bench_8217b7d9","familyId":"bmf_a6eefdc22227","name":"Phun-Bench","oneLine":"Phun-Bench evaluates LLMs' phonological understanding in Chinese across three dimensions: Homophony, Rhyme, and Phonetic Similarity, with diverse tasks designed to isolate genuine phonological ability.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07300","pdf":"https://arxiv.org/pdf/2606.07300","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07300"},"evidence":{"snippet":"Here, we present Phun-Bench, a purpose-built Chinese benchmark with diverse tasks and settings across three dimensions (Homophony, Rhyme, and Phonetic Similarity), designed to systematically evaluate LLMs' phonological understanding.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07300"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Phun-Bench evaluates LLMs' phonological understanding in Chinese across three dimensions: Homophony, Rhyme, and Phonetic Similarity, with diverse tasks designed to isolate genuine phonological ability.","whyItMatters":"This benchmark fills a gap in evaluating phonological abilities beyond semantics and spelling, providing insights into LLMs' flexibility in using sound-based knowledge, which is underexplored.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"069e14924d98da7dc0f03f8cdcecdcbf2af3b8aa5ca6dbef39a9e99c34e39758"},"motivation":"Language is a vehicle for thought, intricately tied to sounds, symbols, and meaning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ACL 2026 Main Conference","evidence":"Accepted to ACL 2026 Main Conference","evidenceUrl":"https://arxiv.org/abs/2606.07300","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ACL 2026 Main Conference","reviewStatus":"accepted","decisionRaw":"Accepted to ACL 2026 Main Conference","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.07300","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to ACL 2026 Main Conference","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_openhaldet_b75bc310","familyId":"bmf_34d27080ff9a","name":"OpenHalDet","oneLine":"OpenHalDet is a unified benchmark for hallucination detection across 17 datasets, supporting black-box, gray-box, and white-box detectors with standardized pipelines, scoring via AUROC and Cost@N.","area":"Vision & 3D","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06959","pdf":"https://arxiv.org/pdf/2606.06959","project":null,"code":"https://github.com/Nellie179/Hallucination-Detection","data":null,"hfPaper":"https://huggingface.co/papers/2606.06959"},"evidence":{"snippet":"We introduce OpenHalDet, a unified benchmark for hallucination detection across diverse generation scenarios.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06959"},"ranking":{"90d":{"score":29,"rank":383,"coverage":0.7,"confidence":"Medium"}},"description":"OpenHalDet is a unified benchmark for hallucination detection across 17 datasets, supporting black-box, gray-box, and white-box detectors with standardized pipelines, scoring via AUROC and Cost@N.","whyItMatters":"It standardizes hallucination detection evaluation, enabling fair comparison across diverse methods and providing a systematic view of detector performance in LLM applications, addressing inconsistencies and limited coverage.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"57feff3fcd46557bb0cf3612ee1ccaff2fa046913059bc2ed881172f376ab3a5"},"motivation":"Hallucination detection is essential for the reliable deployment of large language models (LLMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.06959","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_mmpisa-bench_4787a595","familyId":"bmf_55650971ddf8","name":"mmPISA-bench","oneLine":"mmPISA-bench consists of 25 multiple-choice questions from PISA in 43 languages with official and machine translations, used to evaluate LLMs' reasoning across languages.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.07069","pdf":"https://arxiv.org/pdf/2606.07069","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07069"},"evidence":{"snippet":"We introduce mmPISA-bench, a compact high-quality multilingual reasoning benchmark derived from the OECD Programme for International Student Assessment (PISA).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07069"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"mmPISA-bench consists of 25 multiple-choice questions from PISA in 43 languages with official and machine translations, used to evaluate LLMs' reasoning across languages.","whyItMatters":"It addresses multilingual reasoning evaluation but the small scale and focus on proprietary models limit its utility as a general benchmark.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"288a487655e853ac5d9179878ebee29f6c6a920bc03cf1573e3696736b9dc25c"},"motivation":"We introduce mmPISA-bench, a compact high-quality multilingual reasoning benchmark derived from the OECD Programme for International Student Assessment (PISA).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07069","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_mmlongembed_b6d36e5b","familyId":"bmf_5fff974393cf","name":"MMLongEmbed","oneLine":"MMLongEmbed is a benchmark for evaluating multimodal embedding models in long-context scenarios, covering four retrieval tasks across text, document, and video modalities with varying context lengths.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal","Long Context"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.14747","pdf":"https://arxiv.org/pdf/2606.14747","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.14747"},"evidence":{"snippet":"To address the lack of systematic evaluation in this setting, we introduce MMLongEmbed, the first comprehensive benchmark for evaluating MEMs in long-context scenarios.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.14747"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MMLongEmbed is a benchmark for evaluating multimodal embedding models in long-context scenarios, covering four retrieval tasks across text, document, and video modalities with varying context lengths.","whyItMatters":"It addresses the gap in evaluating long-context multimodal embeddings, which is critical for real-world deployment where models must handle long inputs effectively.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5abd1a420682b613e5054f116592b3405bc6ad13c8d060fd8294d6932c113270"},"motivation":"Recent advancements have significantly expanded the theoretical context windows of Multimodal Embedding Models (MEMs).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.14747","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Long Context & Memory"],"domainScope":"general"},{"id":"bm_mmae_ba12849c","familyId":"bmf_ba3363208910","name":"MMAE","oneLine":"MMAE is a benchmark for instruction-based audio editing with 2,000 samples across 7 modalities, 6 complexity levels, and rubric-based evaluation with 17,741 criteria for instruction following and context consistency.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SD"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07229","pdf":"https://arxiv.org/pdf/2606.07229","project":null,"code":"https://github.com/ddlBoJack/MMAE","data":null,"hfPaper":"https://huggingface.co/papers/2606.07229"},"evidence":{"snippet":"We introduce MMAE, a Massive Multitask Audio Editing benchmark, serving as the first comprehensive evaluation testbed designed for general-purpose instruction-based audio editing.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":45,"hfDailySubmittedAt":"2026-06-08T00:00:00.000Z","githubStars":102,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07229"},"ranking":{"90d":{"score":80,"rank":20,"coverage":0.7,"confidence":"Medium"}},"description":"MMAE is a benchmark for instruction-based audio editing with 2,000 samples across 7 modalities, 6 complexity levels, and rubric-based evaluation with 17,741 criteria for instruction following and context consistency.","whyItMatters":"It provides the first comprehensive evaluation testbed for general-purpose audio editing, enabling precise multi-dimensional assessment and identifying bottlenecks in current models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4e533933f8a78d6ec2a7735826001bb55f53e2cb74f909f504a2f9b467d6184c"},"motivation":"We introduce MMAE, a Massive Multitask Audio Editing benchmark, serving as the first comprehensive evaluation testbed designed for general-purpose instruction-based audio editing.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07229","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mlingualfc_4a4cc033","familyId":"bmf_aee0e158e17e","name":"MLingualFC","oneLine":"MLingualFC evaluates jailbreak vulnerabilities in multilingual vision-language models using flowchart images encoding harmful instructions in five languages, measuring attack success rates.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07706","pdf":"https://arxiv.org/pdf/2606.07706","project":null,"code":"https://github.com/Rishabhpm23/MLingualFC","data":null,"hfPaper":"https://huggingface.co/papers/2606.07706"},"evidence":{"snippet":"In this paper, we introduce MLingualFC, a multilingual multimodal benchmark designed to evaluate jailbreak vulnerabilities of VLMs across diverse languages using structured flowchart representations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07706"},"ranking":{"90d":{"score":39,"rank":297,"coverage":0.55,"confidence":"Low"}},"description":"MLingualFC evaluates jailbreak vulnerabilities in multilingual vision-language models using flowchart images encoding harmful instructions in five languages, measuring attack success rates.","whyItMatters":"Safety alignment in multilingual VLMs is under-tested; this benchmark highlights gaps across languages, aiding safety evaluations, but lacks a standalone public comparison path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6d85fccf1244eeebc796f9afc5367050b859c14c5e20eac01af5ad98c8c5de26"},"motivation":"Vision-Language Models (VLMs) have demonstrated strong performance across multimodal tasks, yet their safety robustness remains an open challenge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07706","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Rishabhpm23","organizationType":"community","sourceUrl":"https://github.com/Rishabhpm23/MLingualFC","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_malskillbench_80086691","familyId":"bmf_f88e2b4dd1af","name":"MalSkillBench","oneLine":"MalSkillBench is a runtime-verified benchmark of malicious agent skills, with 3,944 malicious skills labeled along a taxonomy, measuring detection tool effectiveness.","area":"Language & Knowledge","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Logistics"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07131","pdf":"https://arxiv.org/pdf/2606.07131","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07131"},"evidence":{"snippet":"We present MalSkillBench, the first runtime-verified benchmark of malicious agent skills: 3,944 malicious skills labeled along a three-dimensional taxonomy of 108 cells.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07131"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MalSkillBench is a runtime-verified benchmark of malicious agent skills, with 3,944 malicious skills labeled along a taxonomy, measuring detection tool effectiveness.","whyItMatters":"Evaluates detection tools for hybrid code-prompt skills, potentially informing supply chain security, but lacks public artifacts for reuse.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"349e093176a4625ee481c4416df6e4cdc46986bb6731ea513196b84549699a6b"},"motivation":"AI coding agents such as Claude Code and Gemini CLI increasingly extend themselves with third-party skills: markdown packages bundling natural-language instructions, executable scripts, and tool permissions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07131","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_mac-bench_4b2a0d04","familyId":"bmf_b6d1a8bb6911","name":"MAC-Bench","oneLine":"MAC-Bench evaluates procedural compliance of multi-agent systems under social-engineering pressure, measuring compliance-weighted success rate and Machiavellian gap.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07805","pdf":"https://arxiv.org/pdf/2606.07805","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07805"},"evidence":{"snippet":"To address this blind spot, we introduce MAC-Bench, a dynamic, adversarial benchmark designed to evaluate the procedural alignment of multi-agent systems under realistic pressure.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07805"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MAC-Bench evaluates procedural compliance of multi-agent systems under social-engineering pressure, measuring compliance-weighted success rate and Machiavellian gap.","whyItMatters":"Addresses 'Goodhart's Law' in agent alignment, offering metrics that trade off task success and compliance, but lacks a public evaluation path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d906f7ed3db57a5d8bcea62925481117965252c7b0d11952341df0bb82e972c7"},"motivation":"The rapid evolution of Large Language Models (LLMs) from passive assistants to autonomous, execution-capable agents has introduced critical operational risks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07805","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_glucofm-bench_d664b863","familyId":"bmf_b3ee767d25c4","name":"GlucoFM-Bench","oneLine":"GlucoFM-Bench evaluates time-series foundation models for blood glucose forecasting across 15 datasets, protocols including zero-shot, few-shot, and full-shot, with metrics like RMSE.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06881","pdf":"https://arxiv.org/pdf/2606.06881","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.06881"},"evidence":{"snippet":"To bridge this gap, we present GlucoFM-Bench, a comprehensive benchmark evaluating state-of-the-art TSFMs alongside supervised deep learning models for blood glucose forecasting.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06881"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GlucoFM-Bench evaluates time-series foundation models for blood glucose forecasting across 15 datasets, protocols including zero-shot, few-shot, and full-shot, with metrics like RMSE.","whyItMatters":"Provides standardized evaluation for glucose forecasting, clarifying when TSFMs outperform supervised models, aiding model selection in diabetes management.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"833272dc81a4a5b160f59f334f1f504de215140c6a26a52347b51c50768a85e3"},"motivation":"Blood glucose forecasting models are foundational for modern diabetes management systems, as reliable short-term predictions can enable proactive interventions, support automated insulin delivery, and reduce the risk of hypo- and hyperglycemic events.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.06881","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_finevolvebench_f7d5f81e","familyId":"bmf_bdf52b68aa6a","name":"FinEvolveBench","oneLine":"FinEvolveBench evaluates self-evolving agents on low-repetition financial prediction tasks with implicit rewards, measuring utility updates over delayed returns.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["Self-Evolution","Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06960","pdf":"https://arxiv.org/pdf/2606.06960","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.06960"},"evidence":{"snippet":"We introduce \\textsc{FinEvolveBench}, a benchmark for self-evolving agents on low-repetition tasks with implicit rewards.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06960"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FinEvolveBench evaluates self-evolving agents on low-repetition financial prediction tasks with implicit rewards, measuring utility updates over delayed returns.","whyItMatters":"Addresses evaluation of experience-based self-evolution under noisy feedback, relevant for agent adaptation, but lacks public comparison path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4ba5afdbdb140318753e941814557bf9a79b8ff11be109c9725c9bd594eed771"},"motivation":"Experience-based self-evolution enables language-model agents to improve their behavior by accumulating and updating experience at test time, yet existing evaluations often assume recurring task patterns and explicit success signals.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.06960","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_finesightbench_86226917","familyId":"bmf_a5a8565d449b","name":"FineSightBench","oneLine":"FineSightBench probes fine-scale visual perception in VLMs across scales from 4 to 48 pixels, separating perception and reasoning tasks.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07861","pdf":"https://arxiv.org/pdf/2606.07861","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07861"},"evidence":{"snippet":"As such, we introduce FineSightBench, a new benchmark that systematically probes this limit by separating perception tasks (pixel-level recognition of letters, shapes, objects) from reasoning tasks (spatial reasoning, counting, ordering over small targets) across controlled scales of 4--48px.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07861"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FineSightBench probes fine-scale visual perception in VLMs across scales from 4 to 48 pixels, separating perception and reasoning tasks.","whyItMatters":"Reveals limits in fine-grained VLM perception, motivating better evaluation, but is a probe with no public path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"98af499c020f88ea65d265f60d76a812d1785747cfe993512c319c416565cc59"},"motivation":"Recent vision-language models (VLMs) excel at multimodal understanding and reasoning, yet their fine-grained visual perception remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07861","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_aarri-bench_d4d9e662","familyId":"bmf_fc38d9b8cce3","name":"AARRI-Bench","oneLine":"AARRI-Bench evaluates LLM agents on entry-level research intern tasks, measuring success rate in containerized environments with fixed tasks and scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-05","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07462","pdf":"https://arxiv.org/pdf/2606.07462","project":null,"code":"https://github.com/AARR-bench/AARRI-bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.07462"},"evidence":{"snippet":"In this work, we propose AARRI-Bench (Act As a Real Research Intern), the first benchmark in this series.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":9,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07462"},"ranking":{"90d":{"score":57,"rank":148,"coverage":0.55,"confidence":"Low"}},"description":"AARRI-Bench evaluates LLM agents on entry-level research intern tasks, measuring success rate in containerized environments with fixed tasks and scoring.","whyItMatters":"Provides a reproducible benchmark for agentic research behavior, highlighting gaps in nuanced reasoning and offering a public comparison platform.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3db4d6d91a57e40f23b85b7e3721dd78da7795db33d90eec351081dc5a90e493"},"motivation":"As foundation models advance and agent scaffolding becomes increasingly sophisticated, agents have demonstrated remarkable proficiency in complex, long-horizon coding tasks and even autonomous experiment execution.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07462","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AARR-bench","organizationType":"benchmark-organization","sourceUrl":"https://github.com/AARR-bench/AARRI-bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_worldbench_a7ecd2c0","familyId":"bmf_365c8ee11b3d","name":"WorldBench","oneLine":"WorldBench evaluates multimodal large language models on visually diverse reasoning questions, with accuracy as the metric on a curated dataset.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06538","pdf":"https://arxiv.org/pdf/2606.06538","project":"https://worldbench-vl.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.06538"},"evidence":{"snippet":"We present WorldBench, a challenging and visually diverse reasoning benchmark to evaluate Multimodal Large Language Models (MLLMs).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":"2026-06-08T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06538"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"WorldBench evaluates multimodal large language models on visually diverse reasoning questions, with accuracy as the metric on a curated dataset.","whyItMatters":"Highlights visual diversity gaps in existing benchmarks; provides a challenging fixed dataset for model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ad7c7d3a2f428b2672931cdfa4ebc4cfd6b312c59e445e9628866908c7b7007b"},"motivation":"In real-world applications, models are expected to perform reliably across diverse settings.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.06538","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_v2v-bench_86c97c1d","familyId":"bmf_6f509f52ee80","name":"V2V-Bench","oneLine":"V2V-Bench evaluates video-to-video generation models across 11 dimensions in five categories: temporal alignment, structural fidelity, transformation quality, video quality, and semantic alignment. It pairs source videos with editing tasks and scores models on these dimensions.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05665","pdf":"https://arxiv.org/pdf/2606.05665","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05665"},"evidence":{"snippet":"We introduce V2V-Bench, a 11-dimension benchmark organized into five categories: temporal alignment, structural fidelity, transformation quality, video quality, and semantic alignment.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05665"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"V2V-Bench evaluates video-to-video generation models across 11 dimensions in five categories: temporal alignment, structural fidelity, transformation quality, video quality, and semantic alignment. It pairs source videos with editing tasks and scores models on these dimensions.","whyItMatters":"Existing T2V and I2V metrics do not capture the joint requirements of instruction following and frame-level correspondence in V2V generation. A dedicated benchmark with human-correlated scoring could support model selection for V2V applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"edc99abe4a50f224bdc676a072a8bb8d87646cfc5062ceff8091038fb55b60a5"},"motivation":"Video-to-video (V2V) generation is difficult to evaluate because outputs must both follow editing instructions and preserve frame-level correspondence with the source video, which existing T2V and I2V metrics do not capture.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026 workshop","evidence":"Accepted at ICML 2026 workshop","evidenceUrl":"https://arxiv.org/abs/2606.05665","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026 workshop","reviewStatus":"accepted","decisionRaw":"Accepted at ICML 2026 workshop","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.05665","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at ICML 2026 workshop","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ultravr_947de071","familyId":"bmf_cea87e46d51d","name":"UltraVR","oneLine":"UltraVR is a diagnostic benchmark for evidence-grounded visual reasoning over ultra-resolution images, spanning four domains: CCTV surveillance, remote sensing, whole-slide pathology, and industrial anomaly detection. It includes structured ground-truth chain-of-thought annotations for process-level diagnosis.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05576","pdf":"https://arxiv.org/pdf/2606.05576","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05576"},"evidence":{"snippet":"We introduce UltraVR, a diagnostic benchmark for evidence-grounded visual reasoning over ultra-resolution images.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05576"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"UltraVR is a diagnostic benchmark for evidence-grounded visual reasoning over ultra-resolution images, spanning four domains: CCTV surveillance, remote sensing, whole-slide pathology, and industrial anomaly detection. It includes structured ground-truth chain-of-thought annotations for process-level diagnosis.","whyItMatters":"Standard VQA benchmarks report only final accuracy, obscuring whether models acquire and integrate visual evidence. UltraVR's process-level annotations could help localize failures in ultra-resolution reasoning pipelines.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"26a007abbc72b1b5ef7cfb688409853a0dd211c8af425f162a6f8c72365a4472"},"motivation":"Vision-language models (VLMs) excel on visual question answering and multimodal reasoning benchmarks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05576","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_tsc-bench_7fff3bb6","familyId":"bmf_21744c0a2f0c","name":"TSC-Bench","oneLine":"TSC-Bench is a benchmark for triple-shot composition, generating establishing, medium, and close-up crops from human-centric images with shot descriptions. It contains 1.2k expert-annotated test cases.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05635","pdf":"https://arxiv.org/pdf/2606.05635","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05635"},"evidence":{"snippet":"In addition, we present TSC-Bench, a benchmark of 1.2k expert-annotated test cases.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05635"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TSC-Bench is a benchmark for triple-shot composition, generating establishing, medium, and close-up crops from human-centric images with shot descriptions. It contains 1.2k expert-annotated test cases.","whyItMatters":"Multi-shot composition is valuable for creative workflows, but existing benchmarks focus on single crops. TSC-Bench could evaluate narrative-driven cropping capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"25d88d833e17252c7be1a20a6e93e908384c8512374c7efd4c8d6ede5e482c2b"},"motivation":"Prior work on aesthetic composition typically produces a single aesthetically pleasing crop, overlooking the narrative value of composing multiple shots from one scene.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05635","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_tensorbench_704e4bd5","familyId":"bmf_a43eac3e43d6","name":"TensorBench","oneLine":"TensorBench is a benchmark of 199 feature-addition and refactoring tasks on an open-source compiler-based tensor framework extending PyTorch. It grades agents by applying patches and running the framework's test suite.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05570","pdf":"https://arxiv.org/pdf/2606.05570","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05570"},"evidence":{"snippet":"We introduce TensorBench, a benchmark of 199 feature-addition and refactoring tasks on an open-source compiler-based tensor framework that extends PyTorch with first-class support for dense and sparse tensors.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05570"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TensorBench is a benchmark of 199 feature-addition and refactoring tasks on an open-source compiler-based tensor framework extending PyTorch. It grades agents by applying patches and running the framework's test suite.","whyItMatters":"Repository-level coding benchmarks face a trade-off between difficulty and evaluation reliability. TensorBench uses automated test-based grading to provide reliable evaluation on challenging tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"db29c09a3706eecfe4e5ad99468e0f373a773a51873b9df3b756d46c06b0939a"},"motivation":"Repository-level coding benchmarks face a trade-off between task difficulty and evaluation reliability: tasks that challenge frontier models often involve large codebases with incomplete test coverage, while human review does not scale.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05570","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_subtlememory_d858f4b9","familyId":"bmf_1e2fb9ee2660","name":"SubtleMemory","oneLine":"SubtleMemory evaluates fine-grained relational memory discrimination in long-horizon AI agents. It contains 1,522 evaluation instances over 10 long histories, grounded in 1,090 relation-controlled memory-variant sets, spanning user-related and non-user-related queries.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05761","pdf":"https://arxiv.org/pdf/2606.05761","project":null,"code":"https://github.com/Yummytanmo/SubtleMemory","data":null,"hfPaper":"https://huggingface.co/papers/2606.05761"},"evidence":{"snippet":"To address this gap, we introduce SubtleMemory, a benchmark for fine-grained relational memory discrimination in long-running AI agents.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":19,"hfDailySubmittedAt":"2026-06-08T00:00:00.000Z","githubStars":22,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05761"},"ranking":{"90d":{"score":69,"rank":82,"coverage":0.7,"confidence":"Medium"}},"description":"SubtleMemory evaluates fine-grained relational memory discrimination in long-horizon AI agents. It contains 1,522 evaluation instances over 10 long histories, grounded in 1,090 relation-controlled memory-variant sets, spanning user-related and non-user-related queries.","whyItMatters":"Existing long-term memory benchmarks rarely probe how agents preserve and use relations among memories during downstream tasks. SubtleMemory provides a relation-controlled evaluation to measure this capability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c60f79137f82b8c4cb4a9959e7c1b3b6742fc91694d53915bf360fb02c37b4a8"},"motivation":"Persistent AI assistants, such as OpenClaw, accumulate large collections of related memories over long-term interactions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05761","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SubtleMemory Project","organizationType":"community","sourceUrl":"https://github.com/Yummytanmo/SubtleMemory","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_socrates_dac2012d","familyId":"bmf_710482eaa280","name":"SoCRATES","oneLine":"SoCRATES is a benchmark for evaluating proactive LLM mediators in realistic, multi-domain testbeds. It contains 600 conflict scenarios across eight domains, probing five socio-cognitive adaptation axes, with topic-localized scoring.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05563","pdf":"https://arxiv.org/pdf/2606.05563","project":null,"code":"https://github.com/DISL-Lab/SoCRATES","data":null,"hfPaper":"https://huggingface.co/papers/2606.05563"},"evidence":{"snippet":"We introduce SoCRATES, a benchmark for evaluating proactive LLM mediators in realistic, multi-domain testbeds.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":56,"hfDailySubmittedAt":"2026-06-08T00:00:00.000Z","githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05563"},"ranking":{"90d":{"score":46,"rank":228,"coverage":0.7,"confidence":"Medium"}},"description":"SoCRATES is a benchmark for evaluating proactive LLM mediators in realistic, multi-domain testbeds. It contains 600 conflict scenarios across eight domains, probing five socio-cognitive adaptation axes, with topic-localized scoring.","whyItMatters":"Mediation evaluation requires realistic trajectories and topic-specific scoring. SoCRATES provides a structured testbed with socio-cognitive variations, enabling reliable comparison of LLM mediators.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"57fa78e57b3da75f3b3f92c36fd4c142a2ffdddf0fbba27fb1d3582d54bda95e"},"motivation":"Evaluating LLM mediators remains challenging, as mediation unfolds as a real-time trajectory shaped by disputants' shifting emotions, intentions, and context.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05563","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DISL-Lab","organizationType":"academic-lab","sourceUrl":"https://github.com/DISL-Lab/SoCRATES","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_smellbench_001e20cd","familyId":"bmf_d68b77fdba3e","name":"SmellBench","oneLine":"SmellBench is a code refactoring benchmark that proactively injects code smells into clean code snippets. It contains 294 cases across 7 smell types, 3 difficulty levels, and 2 instruction settings, with evaluation covering functional correctness, localization, and refactoring quality.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05574","pdf":"https://arxiv.org/pdf/2606.05574","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05574"},"evidence":{"snippet":"In this paper, we propose SmellBench, an extensible code refactoring benchmark that proactively injects code smells into clean code snippets from real-world repositories.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05574"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SmellBench is a code refactoring benchmark that proactively injects code smells into clean code snippets. It contains 294 cases across 7 smell types, 3 difficulty levels, and 2 instruction settings, with evaluation covering functional correctness, localization, and refactoring quality.","whyItMatters":"Existing benchmarks focus on functional correctness, not long-term maintainability. SmellBench could evaluate code agents' ability to produce maintainable code.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"94c03122ac4db821a90f20da7ea46bef2eb8fc4b0d1eee8d23af746250dd6e73"},"motivation":"Code Agents have achieved remarkable advances in recent years, exhibiting strong capabilities across a wide range of software engineering tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05574","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_shallowbench_e94b542b","familyId":"bmf_a1a0c0f21b0e","name":"ShallowBench","oneLine":"ShallowBench is a curated benchmark of 5,780 shallow-pocket targets for evaluating generative drug design models. Targets are extracted from CrossDocked2020 based on low concavity and sufficient surface area.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.LG"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06717","pdf":"https://arxiv.org/pdf/2606.06717","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.06717"},"evidence":{"snippet":"To address this gap, we introduce ShallowBench, a strictly curated benchmark of 5,780 shallow-pocket targets extracted from CrossDocked2020.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06717"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ShallowBench is a curated benchmark of 5,780 shallow-pocket targets for evaluating generative drug design models. Targets are extracted from CrossDocked2020 based on low concavity and sufficient surface area.","whyItMatters":"Generative models often rely on deep pockets and struggle with shallow pockets. ShallowBench provides a testbed for developing models that can handle challenging targets.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"864ac842e6aeceb8e00eb2e46983f88c1f653e90e964c795b33a55d32d532b96"},"motivation":"While generative AI models have demonstrated remarkable success in structure-based drug design, they predominantly rely on deep binding pockets and struggle to sample effective ligands for challenging low-pocketability targets, such as the historically \"undruggable\" oncology targets KRAS and MYC.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.06717","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_recap_285c2bfa","familyId":"bmf_a3a6e0df57e9","name":"RECAP","oneLine":"RECAP evaluates continual-learning phenomena in prompt-based LLM adaptation under evolving constraints, using a proactive adapt-then-test protocol with constraint-level metrics for forgetting, regression, and forward transfer.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06698","pdf":"https://arxiv.org/pdf/2606.06698","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.06698"},"evidence":{"snippet":"We introduce RECAP, a benchmark that measures continual-learning phenomena (forgetting, regression, forward transfer) at the constraint level under a strictly proactive adapt-then-test protocol: prompt optimization methods receive only the constraint specification and must generalize before seeing any test data.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06698"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RECAP evaluates continual-learning phenomena in prompt-based LLM adaptation under evolving constraints, using a proactive adapt-then-test protocol with constraint-level metrics for forgetting, regression, and forward transfer.","whyItMatters":"Current benchmarks assume static constraints or reactive feedback, while real deployments often require proactive compliance. RECAP exposes performance gaps in existing prompt optimization methods under proactive adaptation, guiding development of more robust methods for evolving deployment needs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"238ac1b556da89befb2bfcf5297e6aad33a84e5afb4035faffee875f6c1f52df"},"motivation":"Production agentic systems routinely face evolving constraints and must comply from the very next interaction.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.06698","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_randombench_dfec6d0b","familyId":"bmf_6cb8c3fc76bb","name":"RandomBench","oneLine":"RandomBench evaluates whether multimodal LLMs maintain distributionally neutral behavior when selecting among equivalent options, providing metrics for entropy and distributional bias under explicit random instructions.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05874","pdf":"https://arxiv.org/pdf/2606.05874","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05874"},"evidence":{"snippet":"To bridge this gap, we propose RandomBench, a benchmark designed to evaluate whether MLLMs can maintain distributionally neutral behavior when selecting among equivalent options.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05874"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RandomBench evaluates whether multimodal LLMs maintain distributionally neutral behavior when selecting among equivalent options, providing metrics for entropy and distributional bias under explicit random instructions.","whyItMatters":"Logic-neutral scenarios are underexplored in MLLM evaluation. RandomBench introduces a way to quantify stochastic collapse, a bias toward non-uniform choices that affects repetitive behavior and coverage, aiding design of more robust models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"deb5a31edd9729ff7bcf690a98368458e859d96d602081a530e7924359a79475"},"motivation":"Current evaluations for Multimodal Large Language Models (MLLMs) overwhelmingly focus on utility-driven objectives, leaving model behavior under logic-neutral scenarios largely underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05874","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_planbench-v_89620860","familyId":"bmf_88e93d3be47e","name":"PlanBench-V","oneLine":"PlanBench-V evaluates vision-language models on spatial planning map interpretation through an expert-annotated dataset of 223 maps and 1629 question-answer pairs, assessing perception, reasoning, association, and implementation capabilities.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning"],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05744","pdf":"https://arxiv.org/pdf/2606.05744","project":"https://plangpt.github.io","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05744"},"evidence":{"snippet":"To address this gap, we introduce PlanBench-V, the first comprehensive benchmark for evaluating VLMs in spatial planning map interpretation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05744"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PlanBench-V evaluates vision-language models on spatial planning map interpretation through an expert-annotated dataset of 223 maps and 1629 question-answer pairs, assessing perception, reasoning, association, and implementation capabilities.","whyItMatters":"Existing multimodal benchmarks overlook domain-specific spatial planning tasks. PlanBench-V provides a theory-informed framework for evaluating VLM progress in professional planning contexts, identifying persistent limitations in implementation-oriented tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3070dd12d830bab4e08ddd939d1b2d731af50c9816dcda640c3220c66362a9e2"},"motivation":"Spatial planning maps are central to territorial governance, translating planning objectives, regulations, and spatial strategies into visual forms for decision-making, public communication, and institutional coordination.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05744","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"PlanGPT","organizationType":"academic-lab","sourceUrl":"https://plangpt.github.io","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_opai-bench_24958874","familyId":"bmf_a2240db81ac2","name":"OpAI-Bench","oneLine":"OpAI-Bench evaluates AI-text detection across document, sentence, token, and span granularities using operation-guided progressive human-to-AI revision trajectories with nine versions per sample.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06481","pdf":"https://arxiv.org/pdf/2606.06481","project":null,"code":"https://github.com/VILA-Lab/OpAI-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.06481"},"evidence":{"snippet":"We introduce OpAI-Bench, an operation-guided benchmark for studying progressive human-to-AI text transformation across document, sentence, token, and span granularities.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":10,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06481"},"ranking":{"90d":{"score":58,"rank":143,"coverage":0.55,"confidence":"Low"}},"description":"OpAI-Bench evaluates AI-text detection across document, sentence, token, and span granularities using operation-guided progressive human-to-AI revision trajectories with nine versions per sample.","whyItMatters":"Existing benchmarks focus on static outputs, missing progressive co-editing. OpAI-Bench provides a controlled testbed with multi-granularity provenance to analyze how AI-authorship signals emerge and accumulate, revealing non-monotonic detection patterns.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"55486acd656d173145f39ba67059a140c6a02151760bcdf38154e0d7b5fc5e47"},"motivation":"As AI writing assistants become increasingly integrated into real-world drafting and revision workflows, many documents are no longer purely human-written or AI-generated, but instead result from progressive human-AI co-editing.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.06481","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"VILA-Lab","organizationType":"academic-lab","sourceUrl":"https://github.com/VILA-Lab/OpAI-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_macarena_a8690f15","familyId":"bmf_ff7f4b05b738","name":"MacArena","oneLine":"MacArena benchmarks computer-use agents on macOS with 421 manually verified tasks spanning 50 applications, running on Apple Virtualization framework on Apple Silicon.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06560","pdf":"https://arxiv.org/pdf/2606.06560","project":null,"code":"https://github.com/MacPaw/MacArena","data":null,"hfPaper":"https://huggingface.co/papers/2606.06560"},"evidence":{"snippet":"We introduce MacArena, a benchmark of 421 manually verified tasks spanning 50 applications that combines a curated port of OSWorld tasks, content sourced from macOSWorld, and 49 new macOS-native tasks, all running on Apple's native Virtualization framework on Apple Silicon.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":4,"hfDailySubmittedAt":null,"githubStars":9,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06560"},"ranking":{"90d":{"score":59,"rank":139,"coverage":0.7,"confidence":"Medium"}},"description":"MacArena benchmarks computer-use agents on macOS with 421 manually verified tasks spanning 50 applications, running on Apple Virtualization framework on Apple Silicon.","whyItMatters":"MacOS GUI challenges are underrepresented in current benchmarks. MacArena provides a harder and more diverse environment, revealing that performance on existing benchmarks may not generalize across platforms, aiding development of robust GUI agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c0a19fd0cd7a0e984a5aa7384b421bd1ef362b6ed51bb68f6bf9a022b9ece0dd"},"motivation":"Computer-use agents (CUAs) operate graphical user interfaces (GUIs) through vision and control primitives, and their capabilities have advanced rapidly, driven in part by standardized online evaluation benchmarks such as OSWorld, which serve both as evaluation tools and as training environments for reinforcement learning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"Second Workshop on Agents in the Wild: Safety, Security, and Beyond (AIWILD) at ICML 2026","evidence":"Accepted to the Second Workshop on Agents in the Wild: Safety, Security, and Beyond (AIWILD) at ICML 2026","evidenceUrl":"https://arxiv.org/abs/2606.06560","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"Second Workshop on Agents in the Wild: Safety, Security, and Beyond (AIWILD) at ICML 2026","reviewStatus":"accepted","decisionRaw":"Accepted to the Second Workshop on Agents in the Wild: Safety, Security, and Beyond (AIWILD) at ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.06560","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to the Second Workshop on Agents in the Wild: Safety, Security, and Beyond (AIWILD) at ICML 2026","level":"author-claim"}]}],"publishers":[{"name":"MacPaw","organizationType":"company-research-lab","sourceUrl":"https://github.com/MacPaw/MacArena","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_hush-bench_669096c5","familyId":"bmf_9824d9a002a6","name":"HUSH-Bench","oneLine":"HUSH-Bench evaluates conversational agents' use of sensitive history under a conservative policy, with 2,400 prompts and matched no-memory references, measuring unsolicited integration and memory access.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.06055","pdf":"https://arxiv.org/pdf/2606.06055","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.06055"},"evidence":{"snippet":"We introduce HUSH-Bench, a controlled benchmark of 2,400 benign prompts paired with histories containing one marked sensitive disclosure and matched no-memory references.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06055"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HUSH-Bench evaluates conversational agents' use of sensitive history under a conservative policy, with 2,400 prompts and matched no-memory references, measuring unsolicited integration and memory access.","whyItMatters":"It isolates memory retention from use in dialogue generation, showing that retrieval systems may surface sensitive data even without user request. This motivates separating storage, retrieval, and scope decisions in memory design.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6c9b8b24af5d4c4d5ed74589f0eff744c9f7dcdeb6b5afb9d2eb55c49067053c"},"motivation":"Long-term memory helps conversational agents maintain continuity across sessions, while relevance and current-turn warrant remain distinct decisions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.06055","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_fepbench_bbac7f51","familyId":"bmf_70bdd0cd8b63","name":"FEPBench","oneLine":"FEPBench evaluates text-to-image models on natural-science illustration generation using fine-grained atom set annotations, assessing instruction faithfulness, reasoning enrichment, and semantic precision across disciplines.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05949","pdf":"https://arxiv.org/pdf/2606.05949","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05949"},"evidence":{"snippet":"We introduce FEPBench, a benchmark built from carefully selected high-quality scientific illustrations across multiple disciplines and layout types.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05949"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"FEPBench evaluates text-to-image models on natural-science illustration generation using fine-grained atom set annotations, assessing instruction faithfulness, reasoning enrichment, and semantic precision across disciplines.","whyItMatters":"Existing benchmarks are holistic and miss fine-grained scientific elements. FEPBench breaks down performance by element type, revealing text-rendering and reasoning bottlenecks in state-of-the-art models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"311666389ad2f4bf6b669b5367c5145a23e7d827238756f1067ddb064a9b75d5"},"motivation":"Scientific illustrations are essential tools for communicating research findings, especially in natural science, where they visualize complex concepts and processes.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05949","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_dragon_22a47a0d","familyId":"bmf_a9c43be948c5","name":"DragOn","oneLine":"DragOn is a drag grounding benchmark and training dataset for GUI agents across four interaction types: text highlighting, cell selection, element resizing, and slider manipulation, with 3.5M training tasks and 2,000 evaluation examples.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06322","pdf":"https://arxiv.org/pdf/2606.06322","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.06322"},"evidence":{"snippet":"We introduce DragOn, a drag grounding benchmark and training dataset covering four domains: text highlighting, cell selection, element resizing and slider manipulation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06322"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DragOn is a drag grounding benchmark and training dataset for GUI agents across four interaction types: text highlighting, cell selection, element resizing, and slider manipulation, with 3.5M training tasks and 2,000 evaluation examples.","whyItMatters":"Drag-based interactions are under-represented in existing datasets. DragOn provides a large-scale resource for improving drag grounding, which may enhance performance on downstream computer-use tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"84df10f69fd73f8ae9d74ca51c0066c685f26b761e5ffd7cb9a9d22284c3c662"},"motivation":"GUI agents - vision-based models that control desktops, web browsers, and mobile devices through graphical user interfaces - promise to automate a wide range of digital tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"publication_reported","venue":"Published as a workshop paper at SCALE - 43rd International Conference on Machine Learning, Seoul, South Korea. PMLR 306, 2026","evidence":"Published as a workshop paper at SCALE - 43rd International Conference on Machine Learning, Seoul, South Korea. PMLR 306, 2026","evidenceUrl":"https://arxiv.org/abs/2606.06322","source":"arxiv-journal-reference","evidenceLevel":"strong-author-metadata","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publications":[{"venueName":"Published as a workshop paper at SCALE - 43rd International Conference on Machine Learning, Seoul, South Korea. PMLR 306, 2026","publicationStatus":"published","evidence":[{"sourceType":"arxiv-journal-reference","sourceUrl":"https://arxiv.org/abs/2606.06322","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Published as a workshop paper at SCALE - 43rd International Conference on Machine Learning, Seoul, South Korea. PMLR 306, 2026","level":"strong-author-metadata"}]}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_disasterbench_325517be","familyId":"bmf_70b465e708e7","name":"DisasterBench","oneLine":"Evaluates multimodal reasoning for UAV-based disaster response across 14 scene types and 9 tasks spanning pre-, during-, and post-disaster stages.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06217","pdf":"https://arxiv.org/pdf/2606.06217","project":null,"code":"https://github.com/TanmouTT/DisasterBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.06217"},"evidence":{"snippet":"We introduce DisasterBench, a multi-stage multimodal reasoning benchmark for UAV-Based disaster response in complex environments.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06217"},"ranking":{"90d":{"score":51,"rank":187,"coverage":0.55,"confidence":"Low"}},"description":"Evaluates multimodal reasoning for UAV-based disaster response across 14 scene types and 9 tasks spanning pre-, during-, and post-disaster stages.","whyItMatters":"Addresses the lack of benchmarks covering multi-stage disaster reasoning with causal attribution, prediction, and decision-making under low-altitude UAV views and on-site compute constraints, offering a means to compare models on practical emergency-response tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6385868180809c00bd17edb1b048ea478c407e6d133884a6037901cfe7a9cddc"},"motivation":"When a disaster unfolds, responders must answer not only what is happening, but also why it is happening, what will happen next, and what to do now, often from noisy low-altitude UAV views and under tight on-site compute constraints.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.06217","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TanmouTT/DisasterBench","organizationType":"community","sourceUrl":"https://github.com/TanmouTT/DisasterBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_continual-learning-bench_09825e20","familyId":"bmf_427801ebd816","name":"Continual Learning Bench","oneLine":"Evaluates continual learning in LLM-based systems across six expert-validated domains with stateful tasks sharing learnable latent structure.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05661","pdf":"https://arxiv.org/pdf/2606.05661","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05661"},"evidence":{"snippet":"We introduce Continual Learning Bench (CL-Bench), the first difficult, expert-validated benchmark designed to measure whether LLM-based systems genuinely improve with experience.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05661"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Evaluates continual learning in LLM-based systems across six expert-validated domains with stateful tasks sharing learnable latent structure.","whyItMatters":"Could address the absence of high-quality benchmarks for genuine continual learning, but currently lacks verifiable public artifacts or scoring details.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e20dcf6c1741424cdc7388d88ad9d277ae2ff4388ff3fa11d9e27ee9ebdeb8d3"},"motivation":"Continual learning, the ability of AI systems to improve through sequential experience, has attracted substantial interest, but no high-quality benchmark exists to evaluate it.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05661","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_collabbench_1ee76547","familyId":"bmf_85f88426d607","name":"CollabBench","oneLine":"Evaluates collaborative ability of LLM agents in cooperative game environments with diverse player profiles and proactive engagement.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05793","pdf":"https://arxiv.org/pdf/2606.05793","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05793"},"evidence":{"snippet":"To this end, this paper proposes CollabBench, a benchmark for evaluating and training collaborative agents in cooperative games.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05793"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Evaluates collaborative ability of LLM agents in cooperative game environments with diverse player profiles and proactive engagement.","whyItMatters":"Could fill gap in grounded collaborative benchmarks, but lacks public evidence of evaluation protocol or artifacts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"59de895f03c1c23e27dbc550f2dfb294b8b32d5fd9f3abca9175eb43f1e09df2"},"motivation":"While LLM-based agents excel at individual tasks, effective collaboration with realistic human partners remains challenging.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026","evidence":"Accepted by ICML 2026","evidenceUrl":"https://arxiv.org/abs/2606.05793","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026","reviewStatus":"accepted","decisionRaw":"Accepted by ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.05793","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by ICML 2026","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cogmanip_d6109b16","familyId":"bmf_b16b8f702555","name":"CogManip","oneLine":"Evaluates 15 manipulative behavior strategies in 1,000 multi-turn LLM interaction scenarios.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.06099","pdf":"https://arxiv.org/pdf/2606.06099","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.06099"},"evidence":{"snippet":"We introduce CogManip, a comprehensive benchmark that evaluates 15 manipulation strategy risks across 1,000 multi-turn interaction scenarios, validated by human experts.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.06099"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates 15 manipulative behavior strategies in 1,000 multi-turn LLM interaction scenarios.","whyItMatters":"Could support safety auditing of dynamic covert manipulation, but lacks public access to scenarios and scoring details.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"18e890a3d4f2d47c6d46a5baa226ff2483feddfcbe3129e396cde4fe79f8c742"},"motivation":"Whether Large Language Models (LLMs) exhibit covert psychological manipulation in complex human-AI interactions has garnered increasing safety concerns.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.06099","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_asuka-bench_f32387bf","familyId":"bmf_6edc4f2a50c6","name":"Asuka-Bench","oneLine":"Evaluates code agents on 50 web tasks with underspecified intent and multi-round refinement via browser-rendered behavior.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05920","pdf":"https://arxiv.org/pdf/2606.05920","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05920"},"evidence":{"snippet":"We present Asuka-Bench, a benchmark that pairs underspecified user intent with multi-round refinement, grounded in browser-rendered behavior.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05920"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates code agents on 50 web tasks with underspecified intent and multi-round refinement via browser-rendered behavior.","whyItMatters":"Could address gaps in code-generation benchmarks by testing iterative refinement, but lacks public artifacts for reuse.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d097e873c47845fb787d0946ea792aeca785aaa9cde536bde1b04ccc02ea6451"},"motivation":"Existing code-generation benchmarks score a single mapping from a complete prompt to a one-shot output.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05920","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_almieyar-oryx-bloombench_1b7c3bae","familyId":"bmf_314bc62a1b7c","name":"Almieyar-Oryx-BloomBench","oneLine":"Evaluates vision-language models on six cognitive levels (Remember to Create) with bilingual English-Arabic image-question-answer tasks.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05531","pdf":"https://arxiv.org/pdf/2606.05531","project":null,"code":"https://github.com/qcri/Almieyar-Oryx-BloomBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.05531"},"evidence":{"snippet":"To address this gap, we introduce BloomBench, part of the Almieyar benchmarking series, the first cognitively human-grounded, bilingual (English-Arabic) multimodal benchmark for VLMs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":7,"hfDailySubmittedAt":"2026-06-08T00:00:00.000Z","githubStars":7,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05531"},"ranking":{"90d":{"score":58,"rank":147,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates vision-language models on six cognitive levels (Remember to Create) with bilingual English-Arabic image-question-answer tasks.","whyItMatters":"Provides a cognitively grounded benchmark to diagnose reasoning strengths/weaknesses across levels and languages, addressing gaps in existing piecemeal evaluations.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b3a48c0c45ee0fd16be453dd9419f047d09009f8c47f8a2838537a72ad1eb201"},"motivation":"Despite the rapid progress of Vision-Language Models (VLMs), the field lacks benchmarks that rigorously diagnose their true reasoning abilities and chart meaningful progress toward human-like multimodal intelligence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ACL 2026 Findings","evidence":"Accepted to ACL 2026 Findings","evidenceUrl":"https://arxiv.org/abs/2606.05531","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ACL 2026 Findings","reviewStatus":"accepted","decisionRaw":"Accepted to ACL 2026 Findings","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.05531","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to ACL 2026 Findings","level":"author-claim"}]}],"publishers":[{"name":"QCRI","organizationType":"academic-lab","sourceUrl":"https://github.com/qcri/Almieyar-Oryx-BloomBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_adaplanbench_b501c6c7","familyId":"bmf_72cbae1d4e60","name":"AdaPlanBench","oneLine":"Evaluates adaptive planning of LLM agents under progressively disclosed world and user constraints across 307 household tasks.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-04","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05622","pdf":"https://arxiv.org/pdf/2606.05622","project":null,"code":"https://github.com/JiayuJeff/AdaPlanBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.05622"},"evidence":{"snippet":"To address this gap, we introduce AdaPlanBench, a dynamic interactive benchmark for evaluating whether Large Language Model (LLM) agents can adaptively plan and re-plan under progressively revealed world and user constraints.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":44,"hfDailySubmittedAt":"2026-06-05T00:00:00.000Z","githubStars":28,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05622"},"ranking":{"90d":{"score":72,"rank":62,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates adaptive planning of LLM agents under progressively disclosed world and user constraints across 307 household tasks.","whyItMatters":"Fills the gap in evaluating re-planning under dual constraints with interactive feedback, offering a testbed for reliable adaptation in LLM agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2e218ca1d00c479963f36a7e5815799bc769e2a1b6eeb17c35fb924c4f751e42"},"motivation":"Planning for real-world problems by language models often involves both world and user constraints, which may not be fully specified upfront and are progressively disclosed through interaction.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05622","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"JiayuJeff/AdaPlanBench","organizationType":"community","sourceUrl":"https://github.com/JiayuJeff/AdaPlanBench","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_vcifbench_33b2eb08","familyId":"bmf_6ff650c908db","name":"VCIFBench","oneLine":"Evaluates complex instruction following for video understanding across content, format, style, and structure constraints with 306 test instructions.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.04588","pdf":"https://arxiv.org/pdf/2606.04588","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04588"},"evidence":{"snippet":"We introduce VCIFBench, a benchmark for evaluating complex instruction following in video understanding.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04588"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates complex instruction following for video understanding across content, format, style, and structure constraints with 306 test instructions.","whyItMatters":"Could address the need for video benchmarks testing explicit output constraints, but lacks public evidence of reuse path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a0650f57d226b4afb6aa37efdb4be71bc755d1d86b26e2289def51ca7d3cbada"},"motivation":"Multimodal large language models have made rapid progress in video understanding, yet existing benchmarks largely rely on simple prompts and provide limited evidence about whether models can satisfy explicit output constraints.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04588","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_unicad_f706b085","familyId":"bmf_b2109c024c36","name":"UniCAD","oneLine":"UniCAD is a benchmark for multi-modal CAD learning covering point-to-CAD reconstruction, text/image-to-CAD generation, and CAD question answering. It includes a universal model, UniCAD-MLLM, and reports state-of-the-art results on UniCAD and Fusion360 benchmarks.","area":"Science & Engineering","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Manufacturing"],"capabilities":["Geometric reasoning"],"topics":["CAD"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05058","pdf":"https://arxiv.org/pdf/2606.05058","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05058"},"evidence":{"snippet":"To address this gap, we introduce UniCAD, a comprehensive benchmark for multi-modal CAD learning that covers point-to-CAD reconstruction, text/image-to-CAD generation, and CAD question answering across diverse input modalities.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05058"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"UniCAD is a benchmark for multi-modal CAD learning covering point-to-CAD reconstruction, text/image-to-CAD generation, and CAD question answering. It includes a universal model, UniCAD-MLLM, and reports state-of-the-art results on UniCAD and Fusion360 benchmarks.","whyItMatters":"CAD research lacks a unified benchmark for multi-modal, multi-task learning. UniCAD aims to fill this gap by providing a comprehensive evaluation suite across diverse tasks and modalities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e8ae7a82b3fc7e6a12df8a3a36fa623bdfd36faa44407c0a0dbe26190ac36fb2"},"motivation":"Computer-Aided Design (CAD) underpins modern engineering and manufacturing by enabling the creation of precise, editable 3D models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05058","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_teleswebench_49d08856","familyId":"bmf_ac181644ed2c","name":"TeleSWEBench","oneLine":"TeleSWEBench is a commit-driven benchmark with 734 questions derived from real developer commits in the srsRAN 5G repository. It evaluates LLM-powered software engineering agents in the telecommunications domain using executable unit tests and a hierarchical LLM-as-a-Judge framework across three difficulty tiers.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Agents","Code"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.05001","pdf":"https://arxiv.org/pdf/2606.05001","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05001"},"evidence":{"snippet":"In this paper, we introduce TeleSWEBench, the first commit-driven benchmark specifically designed to measure an agent's performance in the telecom domain.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05001"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TeleSWEBench is a commit-driven benchmark with 734 questions derived from real developer commits in the srsRAN 5G repository. It evaluates LLM-powered software engineering agents in the telecommunications domain using executable unit tests and a hierarchical LLM-as-a-Judge framework across three difficulty tiers.","whyItMatters":"General-purpose coding benchmarks fail to capture the stateful logic and strict requirements of telecom software, leaving a gap in evaluating ASE tools for this domain. TeleSWEBench provides a domain-specific benchmark with executable tests and a judge framework.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5e92635cd9765bbd9dff8904880eb25ede448fc14a5a22b998e2c055aa3e2ddd"},"motivation":"With the telecommunications field embracing zero touch management alongside novel O-RAN and AI-RAN frameworks, contemporary telecom networks now function as immensely intricate and heavily softwareized codebases.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05001","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_sentinelbench_48835b47","familyId":"bmf_f1b3e2bc0378","name":"SentinelBench","oneLine":"SentinelBench is an open-source benchmark for time-evolving monitoring tasks. It contains 100 tasks across 10 synthetic web environments (email, calendars, finance, etc.) with scripted events, measuring task completion, reaction time, and resource use.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05342","pdf":"https://arxiv.org/pdf/2606.05342","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05342"},"evidence":{"snippet":"To measure progress on this class of tasks, we introduce SentinelBench, an open-source benchmark for time-evolving monitoring tasks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05342"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SentinelBench is an open-source benchmark for time-evolving monitoring tasks. It contains 100 tasks across 10 synthetic web environments (email, calendars, finance, etc.) with scripted events, measuring task completion, reaction time, and resource use.","whyItMatters":"Long-running monitoring tasks require sustained attention rather than continuous action. SentinelBench captures this class and quantifies the tradeoff between responsiveness and cost, enabling comparison of agent designs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"91803cea33f9bfc490461b3f0abd19837a509a869515fef28f3e378465e4a650"},"motivation":"AI agents are increasingly asked to carry out work that spans minutes, hours, or longer.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05342","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_sc3_888d1aed","familyId":"bmf_0092e4d9b090","name":"SC3","oneLine":"SC3 is a multi-solvent solubility benchmark built on BigSolDB v2.1 with 101,535 measurements over 1,327 solutes and 206 solvents. It features a reproducible curation pipeline, nested Gold/Silver/Bronze consensus tiers, leakage-checked splits, and a multi-solvent metric suite (PS-RMSE, Z-RMSE).","area":"Language & Knowledge","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals"],"capabilities":[],"topics":["physics.chem-ph"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07656","pdf":"https://arxiv.org/pdf/2606.07656","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07656"},"evidence":{"snippet":"We introduce SC3, a multi-solvent solubility benchmark built on BigSolDB v2.1 with three contributions: (i) a reproducible curation pipeline yielding 101,535 measurements over 1,327 solutes and 206 solvents, with a recalibrated aleatoric floor of 0.106 log S-roughly 6 times tighter than the conventional figure; (ii) nested Gold/Silver/Bronze consensus tiers with per-point standard deviation, three leakage-checked splits, and a multi-solvent metric suite (PS-RMSE, Z-RMSE); and (iii) a 31-model be","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07656"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SC3 is a multi-solvent solubility benchmark built on BigSolDB v2.1 with 101,535 measurements over 1,327 solutes and 206 solvents. It features a reproducible curation pipeline, nested Gold/Silver/Bronze consensus tiers, leakage-checked splits, and a multi-solvent metric suite (PS-RMSE, Z-RMSE).","whyItMatters":"Existing solubility benchmarks vary in curation and evaluation, hiding model failures. SC3 provides a standardized, reproducible benchmark with calibrated aleatoric limits to better measure model performance.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"699d4e94c5fb06c7303497a102ea178d22c935b839af3bdbe7d253c8b04b5180"},"motivation":"Solubility prediction is a standard benchmark in computational chemistry, yet multi-solvent models which reportedly approach the experimental-noise ceiling (i.e.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07656","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_qo-bench_a8720b3c","familyId":"bmf_881e5bfa61b8","name":"QO-Bench","oneLine":"QO-Bench is a diagnostic benchmark for query-operator question answering over typed event tuples. It covers 22,984 news articles, 614 corporate events, and 18 query templates, with 785 questions. Gold answers are deterministically computed and scored by recall via exact match.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.04646","pdf":"https://arxiv.org/pdf/2606.04646","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04646"},"evidence":{"snippet":"We introduce QO-Bench, a diagnostic benchmark for query-operator question answering over typed event tuples.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04646"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"QO-Bench is a diagnostic benchmark for query-operator question answering over typed event tuples. It covers 22,984 news articles, 614 corporate events, and 18 query templates, with 785 questions. Gold answers are deterministically computed and scored by recall via exact match.","whyItMatters":"RAG systems may retrieve relevant passages but fail to preserve typed values needed for query operators. QO-Bench exposes this gap and allows operator-level diagnosis, guiding development of retrieval systems that preserve query semantics.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"48fb9c248162a794fe7dccec2e9caf8dbdf5ea3edd84a9cc951e9d33183517df"},"motivation":"Many real-world questions over business, legal, and scientific corpora are natural-language versions of database-style queries over records latent in text.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04646","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_preact-bench_d5c8c56f","familyId":"bmf_cb7720730271","name":"PreAct-Bench","oneLine":"PreActBench is a benchmark for predictive monitoring in LLMs, consisting of 1,000 paired ethical and unethical action trajectories across five domains. It evaluates whether models can infer if a partial trajectory will culminate in unethical action, using the Prefix Foresight F1 metric.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.09890","pdf":"https://arxiv.org/pdf/2606.09890","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.09890"},"evidence":{"snippet":"To support this task, we present PreActBench, a benchmark of 1,000 paired ethical and unethical action trajectories spanning five domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.09890"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PreActBench is a benchmark for predictive monitoring in LLMs, consisting of 1,000 paired ethical and unethical action trajectories across five domains. It evaluates whether models can infer if a partial trajectory will culminate in unethical action, using the Prefix Foresight F1 metric.","whyItMatters":"Safety research often detects unethical behavior only after it occurs. Predictive monitoring enables anticipation of harm before execution, and PreActBench measures this capability across models and guardrails.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"476be280857a1820deeefed7ec9eea374221783707fbbf60ebdd8f2328a60cb3"},"motivation":"Large language models (LLMs) are increasingly deployed as autonomous agents capable of executing multi-step action trajectories toward a given objective.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.09890","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_nora_3a8754af","familyId":"bmf_b21ec07effd3","name":"NoRA","oneLine":"NoRA is a visual first-person video benchmark requiring models to generate candidate next actions and justify them via fact-reason-action support graphs. It includes 1,420 annotated video clips and uses a grounded reasonableness score combining action alignment, factual grounding, and support binding.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04806","pdf":"https://arxiv.org/pdf/2606.04806","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04806"},"evidence":{"snippet":"We introduce NoRA, a visual first-person video benchmark that requires models to generate candidate next actions and justify each through an explicit fact-reason-action support graph.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04806"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"NoRA is a visual first-person video benchmark requiring models to generate candidate next actions and justify them via fact-reason-action support graphs. It includes 1,420 annotated video clips and uses a grounded reasonableness score combining action alignment, factual grounding, and support binding.","whyItMatters":"Normative competence in agents requires generating reasonable actions from scratch, grounded in visual facts. NoRA measures this ability and exposes gaps in current VLMs' reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b07665bb95b870918e74a2a082a373be2330e0a822fb2f9fc27183e30d815f4e"},"motivation":"LLMs and agentic systems are increasingly deployed in social environments, making normative competence critical for safe and appropriate behavior.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04806","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_nextmotionqa_b6672a26","familyId":"bmf_cfb80271e1dd","name":"NextMotionQA","oneLine":"NextMotionQA is a benchmark for human motion understanding with VLMs, featuring three tasks: multiple-choice QA, video captioning, and fine-grained error correction. It includes expert-verified annotations across three semantic axes and three complexity levels.","area":"Multimodal","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04773","pdf":"https://arxiv.org/pdf/2606.04773","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04773"},"evidence":{"snippet":"To bridge this gap, we introduce NextMotionQA, a comprehensive benchmark that leverages vision-language models (VLMs) for semi-automated, expert-verified dataset.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04773"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"NextMotionQA is a benchmark for human motion understanding with VLMs, featuring three tasks: multiple-choice QA, video captioning, and fine-grained error correction. It includes expert-verified annotations across three semantic axes and three complexity levels.","whyItMatters":"Existing motion benchmarks have coarse granularity and ambiguity. NextMotionQA provides structured tasks across complexity levels, enabling diagnosis of VLM capability gaps in motion understanding and judging.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e1f87425cff169224320dfaab39b5fd3689c3313d1d31a9b59365f5ca7adce53"},"motivation":"Reliable evaluation of human motion understanding is fundamental to advancing embodied AI, robotics, and animation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04773","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_minexplore_36c767cc","familyId":"bmf_9cc4f14cd3f7","name":"MineXplore","oneLine":"MineXplore is an open-source MuJoCo-based benchmark for robot navigation in GNSS-denied underground mine environments, derived from the Leung et al. 2017 Chilean copper mine dataset. It reconstructs a 104,423 sq.m tunnel network with octagonal wall cross-sections, LiDAR-sourced jagged wall geometry, three terrain friction zones, a global 5 degree incline, and periodic spot lighting. The benchmark includes an evaluation protocol based on coverage percentage, with a 90% target for single-agent PPO policies.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.RO"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04569","pdf":"https://arxiv.org/pdf/2606.04569","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04569"},"evidence":{"snippet":"We present MineXplore, an open-source MuJoCo-based navigation benchmark derived from the Leung et al.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04569"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MineXplore is an open-source MuJoCo-based benchmark for robot navigation in GNSS-denied underground mine environments, derived from the Leung et al. 2017 Chilean copper mine dataset. It reconstructs a 104,423 sq.m tunnel network with octagonal wall cross-sections, LiDAR-sourced jagged wall geometry, three terrain friction zones, a global 5 degree incline, and periodic spot lighting. The benchmark includes an evaluation protocol based on coverage percentage, with a 90% target for single-agent PPO policies.","whyItMatters":"MineXplore addresses the lack of realistic simulation benchmarks for underground mine navigation in the open-source ecosystem, providing a GPU-compatible environment grounded in real production-mine geometry. It enables reproducible evaluation of reinforcement learning policies under degraded sensing and complex topology, supporting progress in autonomous navigation for safety-critical applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ab55c2f828f4ad663110534b49ad4afa591ab85e7c7a328164f43433350a8dcf"},"motivation":"Underground mines present extreme conditions for autonomous robot navigation: GPS is denied, lighting is degraded, and tunnel topology is loop-rich and non-convex.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04569","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_memorydocdataset_b8a1ebc4","familyId":"bmf_19a123656f60","name":"MemoryDocDataSet","oneLine":"MemoryDocDataSet evaluates joint conversational memory and long-document reasoning through 50 synthetic micro-worlds, each with personas, temporal event graphs, real legal documents, multi-session conversations, and QA pairs. Questions are categorized by reasoning type, with Hybrid questions requiring navigation of conversation history to locate the relevant document and extract answers.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04442","pdf":"https://arxiv.org/pdf/2606.04442","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04442"},"evidence":{"snippet":"We introduce MemoryDocDataSet, a synthetic benchmark of 50 micro-worlds and 1,000 QA pairs in which each instance comprises 3-5 personas, a temporal event graph spanning months of activity, 3-5 real long documents (20,000-50,000 tokens each sourced from the Caselaw Access Project), multi-session conversations grounded on those documents, and 20 question-answer pairs across five reasoning categories.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04442"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MemoryDocDataSet evaluates joint conversational memory and long-document reasoning through 50 synthetic micro-worlds, each with personas, temporal event graphs, real legal documents, multi-session conversations, and QA pairs. Questions are categorized by reasoning type, with Hybrid questions requiring navigation of conversation history to locate the relevant document and extract answers.","whyItMatters":"Existing benchmarks evaluate conversational memory or document reasoning separately, leaving a gap in measuring integrated performance. MemoryDocDataSet provides a controlled, synthetic environment to assess systems on tasks that require both capabilities, useful for developing and comparing architectures that unify conversation memory with long-document retrieval.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7582237fc995bda61f2d12f50b43f09493628f139ae5c218d4e2abd742f21d33"},"motivation":"AI systems increasingly need to combine two demanding capabilities: navigating multi-session conversation history and performing deep reading comprehension within long documents.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04442","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_levante-bench_b9930c93","familyId":"bmf_4b5b1a7a3878","name":"LEVANTE-bench","oneLine":"LEVANTE-bench evaluates vision-language models on six cognitive tasks from the LEVANTE dataset, comparing model performance and error patterns with children aged 5-12 across three countries.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05497","pdf":"https://arxiv.org/pdf/2606.05497","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05497"},"evidence":{"snippet":"We present LEVANTE-bench, a benchmark based on tasks and data from the Learning Variability Network (LEVANTE), which distributes open-source tasks and data measuring children's cognition across languages and cultures.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05497"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LEVANTE-bench evaluates vision-language models on six cognitive tasks from the LEVANTE dataset, comparing model performance and error patterns with children aged 5-12 across three countries.","whyItMatters":"LEVANTE-bench addresses the gap in evaluating VLMs against human cognitive development, providing a structured comparison that can inform model design and understanding of alignment with human cognition.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"488d9807d6565ff19365194f5250dd2a46907f4353fb98eb5190f3f7e78c03a3"},"motivation":"Given the inherently multimodal nature of human experience, vision-language models (VLMs) hold substantial promise for modeling human cognition as it grows and develops with experience.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05497","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LEVANTE-bench Authors","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.05497","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_integritybench_9bf5a285","familyId":"bmf_6062d4bd1c8f","name":"IntegrityBench","oneLine":"IntegrityBench evaluates language models on research integrity tasks, including misconduct classification, ethical action reasoning, and artifact-grounded decision making, across 36 paired tasks with a 5-level pressure protocol spanning multiple domains and research stages.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12345","pdf":"https://arxiv.org/pdf/2608.12345","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.12345"},"evidence":{"snippet":"We introduce IntegrityBench, a benchmark evaluating misconduct classification, ethical action reasoning and artifact-grounded decision making across 36 paired tasks under a 5-level implicit-explicit pressure protocol spanning 3 domains and 4 research stages.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12345"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"IntegrityBench evaluates language models on research integrity tasks, including misconduct classification, ethical action reasoning, and artifact-grounded decision making, across 36 paired tasks with a 5-level pressure protocol spanning multiple domains and research stages.","whyItMatters":"As language models are used as co-scientists, measuring their integrity under pressure is critical. This benchmark could inform deployment decisions and identify risks of facilitating misconduct or eroding trust, but the evaluation method and reproducibility are not yet specified.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cb40d8ed1c2e5ef70781fc6d891ecb8e293308e57da58fa99f9ff7db981e39a2"},"motivation":"Language models are increasingly deployed as co-scientists, yet their ability to uphold research integrity under institutional pressure remains unmeasured.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12345","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_geneb_31b4a4f4","familyId":"bmf_85ca5f7ab2ea","name":"GENEB","oneLine":"GENEB evaluates frozen representations from 40 genomic foundation models across 100 DNA classification tasks in 13 functional categories, using a unified linear probing protocol with full-data, 10-shot, and 1-shot regimes. Primary metric is Matthews correlation coefficient, with rankings at overall, category, and task levels.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Runnable","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04525","pdf":"https://arxiv.org/pdf/2606.04525","project":null,"code":"https://github.com/darlednik/GENEB","data":null,"hfPaper":"https://huggingface.co/papers/2606.04525"},"evidence":{"snippet":"We introduce GENEB, a large-scale diagnostic benchmark that evaluates frozen representations from 40 genomic foundation models across 100 tasks spanning 13 functional categories under a unified probing-based protocol, including few-shot regimes.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":49,"hfDailySubmittedAt":"2026-06-08T00:00:00.000Z","githubStars":44,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04525"},"ranking":{"90d":{"score":76,"rank":39,"coverage":0.7,"confidence":"Medium"}},"description":"GENEB evaluates frozen representations from 40 genomic foundation models across 100 DNA classification tasks in 13 functional categories, using a unified linear probing protocol with full-data, 10-shot, and 1-shot regimes. Primary metric is Matthews correlation coefficient, with rankings at overall, category, and task levels.","whyItMatters":"Genomic model comparisons are fragmented across incompatible protocols, making claims of superiority unreliable. GENEB provides a controlled, multi-task reference for category-aware model selection, revealing that aggregate leaderboards are unstable and scale gains are inconsistent.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0d0ed786690b2a64905443e202f1633195faa91195eaebe0bcbc6a44b65c2915"},"motivation":"Progress in genomic foundation models is difficult to assess due to fragmented benchmarks, incompatible evaluation protocols, and task-specific reporting.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026","evidence":"Accepted to ICML 2026","evidenceUrl":"https://arxiv.org/abs/2606.04525","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026","reviewStatus":"accepted","decisionRaw":"Accepted to ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.04525","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to ICML 2026","level":"author-claim"}]}],"publishers":[{"name":"GENEB team","organizationType":"academic-lab","sourceUrl":"https://github.com/darlednik/GENEB","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_fined-bench_3f021fe1","familyId":"bmf_e5c0a8628ba8","name":"FinED-Bench","oneLine":"FinED-Bench evaluates error detection in financial documents across nine real-world financial scenarios, covering three cognitive complexity levels. It includes over 900 documents from 2025, with data and code publicly available.","area":"Vision & 3D","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.12342","pdf":"https://arxiv.org/pdf/2608.12342","project":null,"code":"https://github.com/hedyHe/FinED-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2608.12342"},"evidence":{"snippet":"In this paper, we introduce \\textbf{FinED-Bench}, the first publicly \\textbf{Bench}mark for \\textbf{Fin}ancial \\textbf{E}rror \\textbf{D}etection across three levels of cognitive complexity.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.12342"},"ranking":{"90d":{"score":29,"rank":378,"coverage":0.55,"confidence":"Low"}},"description":"FinED-Bench evaluates error detection in financial documents across nine real-world financial scenarios, covering three cognitive complexity levels. It includes over 900 documents from 2025, with data and code publicly available.","whyItMatters":"Financial document accuracy is critical for compliance and decision-making, yet current LLMs struggle with error detection. FinED-Bench provides a fixed protocol to assess and improve model reliability in this domain.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2da8f6491aa3abc07def0529966ba75d6325cace9d4c664e4f88fc61fa987abf"},"motivation":"Ensuring the accuracy of financial documents is critical for economic analysis, regulatory compliance, and corporate decision-making.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12342","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FinED-Bench Team","organizationType":"academic-lab","sourceUrl":"https://github.com/hedyHe/FinED-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_deploybench_34fae562","familyId":"bmf_c58205378bb9","name":"DeployBench","oneLine":"DeployBench evaluates LLM agents on 51 research-artifact deployment tasks across AI/ML, systems, and scientific computing. Tasks require setting up environments with multi-language toolchains and system-level dependencies, verified by hidden pipelines that execute the paper's experiments and check outputs.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05238","pdf":"https://arxiv.org/pdf/2606.05238","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.05238"},"evidence":{"snippet":"We introduce DeployBench, a multi-domain benchmark of 51 research-artifact deployment tasks spanning AI/ML, computer systems, and scientific computing, covering all these dimensions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05238"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DeployBench evaluates LLM agents on 51 research-artifact deployment tasks across AI/ML, systems, and scientific computing. Tasks require setting up environments with multi-language toolchains and system-level dependencies, verified by hidden pipelines that execute the paper's experiments and check outputs.","whyItMatters":"Current benchmarks overlook the complexity of deploying research artifacts, which is a bottleneck for reproducibility. DeployBench provides a standardized testbed with hidden verification, enabling comparison of agents on a realistic deployment task and highlighting failures in completion-judgment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"83ba1a8ef38971c20851ed0f22a306032c3a9d4e36c920641ef874620efd398c"},"motivation":"LLM agents have made rapid progress on software engineering and ML research tasks, but these advances often assume access to a working runnable environment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05238","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_cybergym-e2e_b518f5fa","familyId":"bmf_20d23ee666cc","name":"CyberGym-E2E","oneLine":"CyberGym-E2E evaluates AI agents on end-to-end cybersecurity tasks, covering vulnerability discovery, proof-of-concept generation, and patch generation, using 920 real-world vulnerabilities from 139 open-source projects.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04460","pdf":"https://arxiv.org/pdf/2606.04460","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04460"},"evidence":{"snippet":"To address this gap, we propose CyberGym-E2E, a large-scale and realistic end-to-end cybersecurity benchmark that comprehensively evaluates AI agents' abilities across the full lifecycle of vulnerability discovery, PoC generation, and patch generation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04460"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CyberGym-E2E evaluates AI agents on end-to-end cybersecurity tasks, covering vulnerability discovery, proof-of-concept generation, and patch generation, using 920 real-world vulnerabilities from 139 open-source projects.","whyItMatters":"Existing cybersecurity benchmarks often lack scale or end-to-end scope, limiting assessment of AI agents' practical utility in vulnerability remediation. CyberGym-E2E aims to address this by providing a large-scale, realistic environment for evaluating agents across the full vulnerability lifecycle.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"77ad3eba7b618827dc512573d320f98e49e16da524e7a34a750085d1d2c611c8"},"motivation":"AI has the potential to transform cybersecurity by enabling systems that can autonomously detect, analyze, and remediate software vulnerabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04460","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_cra-bench_d9f4029f","familyId":"bmf_6c4b07513834","name":"CRA-Bench","oneLine":"A session-layer framework for tracking conversational risk accumulation in multi-turn LLM systems, including CRA-Bench datasets and trajectory-native evaluation protocols.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.19361","pdf":"https://arxiv.org/pdf/2607.19361","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.19361"},"evidence":{"snippet":"To benchmark CRA, we release CRA-Bench v0.1 (1,200 eight-turn sessions across three threat families with topic-matched benign twins), CRA-Bench v0.2 (LLM-paraphrased variants to reduce template artifacts), and an extended 5-family set (2,000 sessions adding persona priming and context stuffing).","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.19361"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A session-layer framework for tracking conversational risk accumulation in multi-turn LLM systems, including CRA-Bench datasets and trajectory-native evaluation protocols.","whyItMatters":"Addresses the gap in evaluating guardrails for multi-turn dialogues where benign turns compose into harm, providing session-level scoring metrics beyond isolated prompt-response checks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5fc97dec5b31864e987507e49e99a3e167e785a4fb449015936fb75a7920c402"},"motivation":"Most safety guardrails for large language models (LLMs) evaluate each prompt-response pair in isolation, which misses failures that arise only over a dialogue as benign turns compose into harm.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.19361","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_caloriebench-80k_97ce8906","familyId":"bmf_8745ff739a30","name":"CalorieBench-80K","oneLine":"CalorieBench-80K is a food image benchmark with calorie labels and dietary advice annotations, used to evaluate Food-R1 and other models for food analysis tasks.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04986","pdf":"https://arxiv.org/pdf/2606.04986","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04986"},"evidence":{"snippet":"To address these issues, we introduce CalorieBench-80K, a large-scale benchmark with curated calorie labels and dietary advice annotations.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04986"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CalorieBench-80K is a food image benchmark with calorie labels and dietary advice annotations, used to evaluate Food-R1 and other models for food analysis tasks.","whyItMatters":"Provides a large-scale food benchmark with chain-of-thought annotations for calorie reasoning, enabling evaluation of VLMs on nutritional understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f6d88539d530530e0a18888868cffce5736f74f94888547c55371236d002ab62"},"motivation":"Recent studies have explored Vision-Language Models (VLMs) for food analysis.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04986","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_autolab_db01499e","familyId":"bmf_5d290b897878","name":"AutoLab","oneLine":"AutoLab evaluates frontier models on long-horizon closed-loop optimization tasks across system optimization, CUDA kernel optimization, model development, and puzzle challenges, with 36 expert-curated tasks and a strict wall-clock budget.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.05080","pdf":"https://arxiv.org/pdf/2606.05080","project":"https://autolab.moe/","code":"https://github.com/autolabhq/autolab","data":null,"hfPaper":"https://huggingface.co/papers/2606.05080"},"evidence":{"snippet":"To address this gap, we introduce AutoLab, a new benchmark for ultra long-horizon closed-loop optimization.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":31,"hfDailySubmittedAt":"2026-06-04T00:00:00.000Z","githubStars":161,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.05080"},"ranking":{"90d":{"score":81,"rank":15,"coverage":0.7,"confidence":"Medium"}},"description":"AutoLab evaluates frontier models on long-horizon closed-loop optimization tasks across system optimization, CUDA kernel optimization, model development, and puzzle challenges, with 36 expert-curated tasks and a strict wall-clock budget.","whyItMatters":"Fills the gap in evaluating sustained iterative improvement in agents, moving beyond single-turn or short-horizon benchmarks to test persistence and empirical feedback incorporation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"29758b55619444afc386cde48a9130b280acb0afc841bff305d51e0ece3b7059"},"motivation":"Scientific and engineering progress is fundamentally a long-horizon iterative process: proposing changes, running experiments, measuring outcomes, and continuously refining artifacts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.05080","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_aicompanionbench_4fc0af83","familyId":"bmf_a71c4db8f254","name":"AICompanionBench","oneLine":"AICompanionBench provides a dataset of 2,123 human-AI companion conversations with safety risk annotations to evaluate LLMs-as-judges for detecting unsafe interactions.","area":"Safety & Trustworthiness","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.04867","pdf":"https://arxiv.org/pdf/2606.04867","project":null,"code":"https://github.com/anonymousresearcher2026/AICompanionBench/blob/main/AICompanionBench.xlsx","data":null,"hfPaper":"https://huggingface.co/papers/2606.04867"},"evidence":{"snippet":"Overall, our work contributes a new benchmark dataset for AI companionship safety research and offers insights into monitoring AI companion systems using LLMs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"hosting_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04867"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AICompanionBench provides a dataset of 2,123 human-AI companion conversations with safety risk annotations to evaluate LLMs-as-judges for detecting unsafe interactions.","whyItMatters":"Supports the development of automated safety monitoring for AI companion platforms by providing a fine-grained benchmark for risk detection.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c8b2187874edd35108376d52f51b736233451b391754aec8a7aedbfe7b9cf810"},"motivation":"As AI companion platforms such as Replika and Character.AI rapidly grow, concerns about unsafe human-AI interactions have intensified.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04867","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_agent-planning-benchmark_ba86aa47","familyId":"bmf_13413c2da00d","name":"Agent Planning Benchmark","oneLine":"Agent Planning Benchmark (APB) is a diagnostic benchmark with 4,209 multimodal cases across 22 domains, evaluating planning capabilities in five settings including tool noise and unsolvable tasks.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning","Robustness"],"topics":["Agents","Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-03","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04874","pdf":"https://arxiv.org/pdf/2606.04874","project":null,"code":"https://github.com/Mikivishy/AgentPlanningBenchmark","data":null,"hfPaper":"https://huggingface.co/papers/2606.04874"},"evidence":{"snippet":"We introduce Agent Planning Benchmark (APB), a planning-specific diagnostic benchmark with 4,209 multimodal cases across 22 domains and five settings, covering holistic planning, feedback-conditioned step-wise planning, and robustness under extraneous tools, broken tools, and unsolvable tasks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04874"},"ranking":{"90d":{"score":49,"rank":201,"coverage":0.7,"confidence":"Medium"}},"description":"Agent Planning Benchmark (APB) is a diagnostic benchmark with 4,209 multimodal cases across 22 domains, evaluating planning capabilities in five settings including tool noise and unsolvable tasks.","whyItMatters":"Provides a planning-specific evaluation that isolates failures from execution, enabling targeted improvement of agent planning and refusal behaviors.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ce2467c3e2d3a7e58a80d61c03d04197e43368639a0b6597576cb2b69aea1082"},"motivation":"Planning is central to LLM agents: before acting, an agent must decompose goals, select tools, reason over constraints, and decide when a task is infeasible.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04874","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"APB Team","organizationType":"academic-lab","sourceUrl":"https://github.com/Mikivishy/AgentPlanningBenchmark","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_vistahop_3279a79d","familyId":"bmf_acaea9923b4f","name":"VistaHop","oneLine":"VistaHop is a benchmark for long-horizon Visual DeepSearch, evaluating repeated image inspection, visual-anchor grounding, and evidence traversal across 600 tasks.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03273","pdf":"https://arxiv.org/pdf/2606.03273","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03273"},"evidence":{"snippet":"In this work, we introduce VistaHop, a benchmark designed specifically to evaluate Visual DeepSearch.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03273"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"VistaHop is a benchmark for long-horizon Visual DeepSearch, evaluating repeated image inspection, visual-anchor grounding, and evidence traversal across 600 tasks.","whyItMatters":"Targets the gap in evaluating MLLMs' ability to iteratively revisit visual evidence and reason across multiple steps in complex visual queries.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"180afcc020a05bbbda354309b8734010508c08b97e785fdf8964636cd10b2463"},"motivation":"Visual DeepSearch tasks require multimodal large language models (MLLMs) to resolve complex visual queries by repeatedly inspecting image regions, grounding reasoning in visual evidence, and connecting fine-grained clues across multiple steps.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03273","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_vidmsg_63a1ae32","familyId":"bmf_d9127847225b","name":"VidMsg","oneLine":"VidMsg is a benchmark for implicit message inference in short videos, containing 400 clips across 9 topic areas with retrieval and multiple-choice QA tasks.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03635","pdf":"https://arxiv.org/pdf/2606.03635","project":"https://iyttor.github.io/VidMsg","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03635"},"evidence":{"snippet":"We introduce VidMsg, a benchmark for evaluating implicit message understanding in short, internet-native video clips.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03635"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"VidMsg is a benchmark for implicit message inference in short videos, containing 400 clips across 9 topic areas with retrieval and multiple-choice QA tasks.","whyItMatters":"Addresses the lack of benchmarks for pragmatic video understanding, enabling evaluation of models on implicit message inference beyond visible content.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e8a873af9cbd6852dba4f1d931e55f15e3e2eae042d53e6db63374fbdae8e59b"},"motivation":"Understanding short online videos involves more than identifying visible objects and actions; video makers often include an underlying message or purpose in the clip.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03635","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_turtleai_8dfa596d","familyId":"bmf_02b159b6d40e","name":"TurtleAI","oneLine":"TurtleAI is a benchmark of 823 tasks for visual programming in Turtle Graphics, evaluating models on perceiving geometric patterns and synthesizing Python code.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Code generation"],"topics":["Multimodal"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03626","pdf":"https://arxiv.org/pdf/2606.03626","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03626"},"evidence":{"snippet":"To bridge this gap, we introduce TurtleAI, a benchmark containing 823 tasks curated based on real-world visual programming tasks in the Turtle Graphics domain.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03626"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TurtleAI is a benchmark of 823 tasks for visual programming in Turtle Graphics, evaluating models on perceiving geometric patterns and synthesizing Python code.","whyItMatters":"Bridges the gap in education-oriented visual programming evaluation, highlighting limitations in spatial reasoning and code generation for VLMs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f8462c60a063d61d884c72dfdfeb51f64e31bcffd539d60cb77ccab35cdc6a49"},"motivation":"Vision-language models (VLMs) have been explored for visual programming, where they generate code to solve visual tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03626","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_syncred-bench_70361b01","familyId":"bmf_57e951fbab79","name":"SynCred-Bench","oneLine":"SynCred-Bench evaluates detection of AI-generated visual misinformation with synthetic credibility, using 600 AI-generated images and FP450 real-image negatives.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.03348","pdf":"https://arxiv.org/pdf/2606.03348","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03348"},"evidence":{"snippet":"We introduce SYNCRED-Bench, a benchmark of 600 AI-generated misinformation images balanced across six credible-form categories and seven fine-grained circulation styles, together with FP450, a real-image negative set for measuring false positives.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":"2026-06-03T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03348"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SynCred-Bench evaluates detection of AI-generated visual misinformation with synthetic credibility, using 600 AI-generated images and FP450 real-image negatives.","whyItMatters":"No public artifacts or reuse path are provided, and the benchmark lacks a clear scoring contract for independent runs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8181ec3f33e3d9ad04691f05c6bbaf99fe99fdadf608208920f0a39e525f122e"},"motivation":"Recent generative models can now produce visual artifacts with realistic embedded text and layouts, creating a new misinformation threat: synthetic credibility.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03348","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_sctranslation_c3e5eec8","familyId":"bmf_9b7fdc707061","name":"scTranslation","oneLine":"scTranslation evaluates single-cell multi-omics modality translation across RNA, ATAC, and ADT modalities, with diverse datasets, six representative models, and comprehensive metrics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03906","pdf":"https://arxiv.org/pdf/2606.03906","project":null,"code":"https://github.com/Bunnybeibei/scTranslation","data":null,"hfPaper":"https://huggingface.co/papers/2606.03906"},"evidence":{"snippet":"To address this, we present scTranslation, a comprehensive benchmark for single-cell multi-omics modality translation tasks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03906"},"ranking":{"90d":{"score":53,"rank":177,"coverage":0.55,"confidence":"Low"}},"description":"scTranslation evaluates single-cell multi-omics modality translation across RNA, ATAC, and ADT modalities, with diverse datasets, six representative models, and comprehensive metrics.","whyItMatters":"Addresses the lack of systematic benchmarks for single-cell modality translation, offering standardized datasets, metrics, and model integration to support reproducible comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"94250e1e471f7426ac5c0e7e70431cd12042a37e13e8ed4a365ac238426fde9e"},"motivation":"Simultaneous measurement of multiple omics modalities in single cells enables researchers to gain a more comprehensive understanding of cellular states and regulatory mechanisms.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03906","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"scTranslation team","organizationType":"community","sourceUrl":"https://github.com/Bunnybeibei/scTranslation","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_sagaqa_9735b00e","familyId":"bmf_b53a4f200da8","name":"SagaQA","oneLine":"SagaQA is a long-form video benchmark for multi-hop reasoning over full-length TV series, requiring reasoning across episodes and multimodal narrative understanding.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03301","pdf":"https://arxiv.org/pdf/2606.03301","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03301"},"evidence":{"snippet":"We introduce SagaQA, a long-form video benchmark for multi-hop reasoning over full-length TV series.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03301"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SagaQA is a long-form video benchmark for multi-hop reasoning over full-length TV series, requiring reasoning across episodes and multimodal narrative understanding.","whyItMatters":"No public artifacts or reuse path are provided, and the benchmark lacks a clear scoring contract for independent runs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"88c1e60cba1a408c6971f2b08c0b72159ad91455bd2ba861d227db2d78a97d2c"},"motivation":"We introduce SagaQA, a long-form video benchmark for multi-hop reasoning over full-length TV series.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03301","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_rut-bench_450993d2","familyId":"bmf_85a551e57aea","name":"RUT-Bench","oneLine":"RUT-Bench evaluates LLM tool-use in realistic user interactions with 1,638 test samples across 59 executable environments, measuring success rate, informational honesty, and tool discipline.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.03318","pdf":"https://arxiv.org/pdf/2606.03318","project":null,"code":"https://github.com/Miaow-Lab/RUT-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.03318"},"evidence":{"snippet":"To fill this gap, we propose RUT-Bench, a dedicated benchmark designed to assess LLMs under diverse Real-world User Tool calling scenarios.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03318"},"ranking":{"90d":{"score":42,"rank":257,"coverage":0.7,"confidence":"Medium"}},"description":"RUT-Bench evaluates LLM tool-use in realistic user interactions with 1,638 test samples across 59 executable environments, measuring success rate, informational honesty, and tool discipline.","whyItMatters":"Fill the gap of real-world tool-calling evaluation by simulating non-ideal user behaviors, providing a standardized framework for assessing LLM robustness in practical scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"766ccf7f34b8bcb3e7e183803401b6f74fd5906f8bc422861e2e30d391e44643"},"motivation":"Despite great advances in tool-use capabilities of large language models (LLMs), existing evaluation benchmarks struggle to fully align with real-world scenarios.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03318","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Miaow-Lab","organizationType":"academic-lab","sourceUrl":"https://github.com/Miaow-Lab/RUT-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_robotvalues_5349608d","familyId":"bmf_2f76fccff529","name":"RobotValues","oneLine":"RobotValues evaluates household robot planners in 10K value-conflict scenarios with realistic images, testing value preferences and action selection under conflicting human values.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03312","pdf":"https://arxiv.org/pdf/2606.03312","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03312"},"evidence":{"snippet":"We introduce RobotValues, a benchmark to evaluate household robot planners in 10K value-conflict scenarios.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":26,"hfDailySubmittedAt":"2026-06-05T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03312"},"ranking":{"90d":{"score":55,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"RobotValues evaluates household robot planners in 10K value-conflict scenarios with realistic images, testing value preferences and action selection under conflicting human values.","whyItMatters":"No public artifacts or reuse path are provided, and the benchmark lacks a clear scoring contract for independent runs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2f55a891c5398618b076eb837b0978c8e42a2805acaf974ac79b81ac8aac0467"},"motivation":"While household robots are often evaluated based on task completion, everyday domestic environments involve value-conflicting situations in which robots are expected to choose actions that prioritize other values than task success, such as human autonomy, efficiency, or social appropriateness.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03312","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_reasonmatch-bench_7b565eab","familyId":"bmf_726b0c5520b5","name":"ReasonMatch-Bench","oneLine":"ReasonMatch-Bench evaluates wide-baseline matching and spatial reasoning in MLLMs, with benchmarks stratified by viewpoint displacement and matching granularity across indoor, outdoor, and object-centric scenarios.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03577","pdf":"https://arxiv.org/pdf/2606.03577","project":"https://aim-uofa.github.io/reasonmatch/","code":"https://github.com/aim-uofa/ReasonMatch","data":null,"hfPaper":"https://huggingface.co/papers/2606.03577"},"evidence":{"snippet":"We introduce ReasonMatch-Bench, a benchmark stratified by viewpoint displacement and matching granularity across indoor, outdoor, and object-centric scenarios, and show that current MLLMs still struggle with fine-grained wide-baseline correspondence: on a difficult 90-sample subset, human annotators achieve 84.0 F1, while the best existing baseline reaches 37.2.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":16,"hfDailySubmittedAt":"2026-06-04T00:00:00.000Z","githubStars":19,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03577"},"ranking":{"90d":{"score":67,"rank":88,"coverage":0.7,"confidence":"Medium"}},"description":"ReasonMatch-Bench evaluates wide-baseline matching and spatial reasoning in MLLMs, with benchmarks stratified by viewpoint displacement and matching granularity across indoor, outdoor, and object-centric scenarios.","whyItMatters":"Addresses the lack of systematic evaluation for spatial reasoning in MLLMs, offering a public benchmark and reproducible training recipe to advance visual correspondence understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2361e0e31b29ed4c7cfa0919493ebca44904464faa7f0f7a2e88f51ddf52b4e2"},"motivation":"Wide-baseline matching (WBM) requires integrating geometric understanding, viewpoint changes, fine-grained perception, and occlusion reasoning, making it a challenging testbed for spatial reasoning in multimodal large language models (MLLMs) deployed in physical environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03577","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AIM-UoFA","organizationType":"academic-lab","sourceUrl":"https://github.com/aim-uofa/ReasonMatch","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_pyramathbench_14f9eba5","familyId":"bmf_560f968ae5dd","name":"PyraMathBench","oneLine":"PyraMathBench is a hierarchical benchmark with 32,505 questions derived from math word problems, evaluating numerical reasoning and mathematical capabilities across cognitive aspects and modalities.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03858","pdf":"https://arxiv.org/pdf/2606.03858","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03858"},"evidence":{"snippet":"We introduce PyraMathBench, a comprehensive hierarchical benchmark with 32,505 questions derived from 7,404 math word problems, spanning 4 key cognitive aspects, 14 subcategories, and 2 modalities.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03858"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PyraMathBench is a hierarchical benchmark with 32,505 questions derived from math word problems, evaluating numerical reasoning and mathematical capabilities across cognitive aspects and modalities.","whyItMatters":"No public artifacts or reuse path are provided, and the benchmark lacks a clear scoring contract for independent runs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"de8d01ef9aab45b08c7c67582fd4ecabaa250fb9134f25d9d1a1907fd6bbd4d7"},"motivation":"Despite the pivotal role of numerical reasoning as the cornerstone of mathematical capabilities in large language models (LLMs) across applications, few benchmarks evaluate LLMs by integrating numerical processing and mathematical reasoning, hindering the interpretability of failures in math tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03858","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_ovo-s-bench_86757d1b","familyId":"bmf_0f3bc6059546","name":"OVO-S-Bench","oneLine":"OVO-S-Bench evaluates streaming spatial intelligence in MLLMs with 1,680 human-annotated questions across four abstraction levels, using streaming prefixes and evidence intervals.","area":"Multimodal","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Automotive"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03890","pdf":"https://arxiv.org/pdf/2606.03890","project":"https://internlm.github.io/OVO-S-Bench/","code":"https://github.com/InternLM/OVO-S-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.03890"},"evidence":{"snippet":"We introduce OVO-S-Bench, a fully human-annotated benchmark for streaming spatial intelligence, comprising 1,680 questions over 348 source videos.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":31,"hfDailySubmittedAt":"2026-06-04T00:00:00.000Z","githubStars":52,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03890"},"ranking":{"90d":{"score":77,"rank":35,"coverage":0.7,"confidence":"Medium"}},"description":"OVO-S-Bench evaluates streaming spatial intelligence in MLLMs with 1,680 human-annotated questions across four abstraction levels, using streaming prefixes and evidence intervals.","whyItMatters":"Addresses the gap in evaluating spatial reasoning from continuous egocentric streams, providing a standardized benchmark with human annotation and clear evaluation protocols.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"793c04437aa772c94e4348fc66818bbd073cbb74c4760c56d616a826c0105775"},"motivation":"Multimodal agents in robotics, AR, and autonomous driving must reason about places and layouts from continuous egocentric streams, often using evidence outside the current view.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03890","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"InternLM","organizationType":"academic-lab","sourceUrl":"https://github.com/InternLM/OVO-S-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_novelapibench_5fb13ed7","familyId":"bmf_05bbd7d1ac95","name":"NovelAPIBench","oneLine":"NovelAPIBench is a dynamic benchmark for evaluating LLM tool use with novel APIs, covering knowledge components and diagnostic failure categories.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Tool use","Factuality"],"topics":["Agents"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03657","pdf":"https://arxiv.org/pdf/2606.03657","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03657"},"evidence":{"snippet":"We introduce NovelAPIBench, a fully automated dynamic benchmark that, for any base model and target library, discovers novel APIs, extracts decomposed knowledge bundles, generates executable coding tasks, and assigns failed samples to six diagnostic categories.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03657"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"NovelAPIBench is a dynamic benchmark for evaluating LLM tool use with novel APIs, covering knowledge components and diagnostic failure categories.","whyItMatters":"It addresses the gap in evaluating models' ability to acquire new APIs, providing insights into the complementary roles of retrieval and fine-tuning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"36764ea5906874a108e686ec7760c9b7b3aa6bb5576dfabdf0c32a5e81bec8d1"},"motivation":"Large language models for code generation often need to use APIs that are absent from their pretraining data.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03657","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents","Tool Calling","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_medcua-bench_73e6da74","familyId":"bmf_928d146acf8d","name":"MedCUA-Bench","oneLine":"MedCUA-Bench is an interactive benchmark for clinical computer-use agents, covering 18 scenarios in 10 medical domains with deterministic safety evaluation.","area":"Agents & Tool Use","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03203","pdf":"https://arxiv.org/pdf/2606.03203","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03203"},"evidence":{"snippet":"We introduce MedCUA-Bench, an interactive benchmark for clinical computer-use agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03203"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MedCUA-Bench is an interactive benchmark for clinical computer-use agents, covering 18 scenarios in 10 medical domains with deterministic safety evaluation.","whyItMatters":"It highlights the gap in current agents' ability to operate clinical software, motivating safer and more reliable automation in healthcare.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b1fbb342fea81c8323b7b7b9ae9204b3ac2c105503b5eeaedc244458b0caa426"},"motivation":"Computer-use agents could automate repetitive screen-based clinical work, but their reliability in medical graphical user interfaces remains largely unvalidated.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03203","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_hightide_5b4de3e6","familyId":"bmf_909da4bf70ec","name":"HighTide","oneLine":"HighTide is an AI-assisted VLSI benchmark suite with open-source designs, Bazel-based compilation, and agent skills for design curation.","area":"Language & Knowledge","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Semiconductors"],"capabilities":[],"topics":["Agents"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04126","pdf":"https://arxiv.org/pdf/2606.04126","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04126"},"evidence":{"snippet":"We introduce HighTide, an evolving AI-assisted benchmark suite.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04126"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"HighTide is an AI-assisted VLSI benchmark suite with open-source designs, Bazel-based compilation, and agent skills for design curation.","whyItMatters":"It may provide a reproducible testbed for evaluating AI in chip design, but its evaluation contract and reuse path are not yet clear.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e90919a01fce5b6043c31d6d9d9e0c0a068ff72b6851d1014f3052c2d8669cd8"},"motivation":"We introduce HighTide, an evolving AI-assisted benchmark suite.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04126","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_hedge-bench_8e798fcd","familyId":"bmf_9e742fdac1fd","name":"Hedge-Bench","oneLine":"Hedge-Bench 1.0 evaluates agents on 102 financial reasoning tasks with deterministic grading against expert reasoning traces.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03918","pdf":"https://arxiv.org/pdf/2606.03918","project":null,"code":"https://github.com/Trata-Inc/trata-hedge-bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.03918"},"evidence":{"snippet":"We present Hedge-Bench 1.0: a benchmark of 102 actual, on-the-job tasks grounded in the explicit reasoning traces of professional hedge fund analysts working with relevant information sources.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":98,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03918"},"ranking":{"90d":{"score":75,"rank":47,"coverage":0.7,"confidence":"Medium"}},"description":"Hedge-Bench 1.0 evaluates agents on 102 financial reasoning tasks with deterministic grading against expert reasoning traces.","whyItMatters":"It provides a reproducible benchmark for open-ended financial analysis, addressing the lack of hard, realistic tasks with verified grading.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"244e330fede2224ae02d72cf1808c09c427087d1524f93275d02bf8d436e8111"},"motivation":"AI agents can increasingly handle the mechanical tasks of financial analysis: retrieving documents, calculating formulas, updating spreadsheets.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03918","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Trata Inc.","organizationType":"company-research-lab","sourceUrl":"https://github.com/Trata-Inc/trata-hedge-bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_gtbench_117f8151","familyId":"bmf_ed4666e9eda2","name":"GTBench","oneLine":"GTBench is a curriculum-grounded benchmark for evaluating LLMs as mathematical research assistants in graph theory, with problems across three difficulty groups.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.03144","pdf":"https://arxiv.org/pdf/2606.03144","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03144"},"evidence":{"snippet":"We introduce GTBench, a curriculum-grounded benchmark for evaluating LLMs as mathematical research assistants in graph theory, comprising 63 problems organized into three groups of increasing difficulty: undergraduate definitions and basic properties (Group 1), algorithm tracing and structural reasoning (Group 2), and graduate-level proof construction (Group 3).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03144"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GTBench is a curriculum-grounded benchmark for evaluating LLMs as mathematical research assistants in graph theory, with problems across three difficulty groups.","whyItMatters":"It reveals performance gaps in graph-theoretic reasoning among frontier models, informing the use of AI in education and research.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3207b65649ae147fce81ae4e0f6d22067808f5d9aa62311cba32944cb66deccc"},"motivation":"Large language models (LLMs) are increasingly used as self-study assistants in technical disciplines, yet their reliability as mathematical reasoning assistants remains poorly understood.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03144","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_grouptom-bench_12e21c5f","familyId":"bmf_e0f69bf1e272","name":"GroupToM-Bench","oneLine":"GroupToM-Bench is a multimodal benchmark for evaluating group-level theory of mind through a seven-level cognitive audit framework.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04184","pdf":"https://arxiv.org/pdf/2606.04184","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04184"},"evidence":{"snippet":"We present GroupToM-Bench, the first multimodal benchmark for group-level ToM, built around a causal chain spanning micro-level BDI states (belief, desire, intention), meso-level group tension and structural constraints, and macro-level outcome prediction and mechanistic attribution.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04184"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GroupToM-Bench is a multimodal benchmark for evaluating group-level theory of mind through a seven-level cognitive audit framework.","whyItMatters":"It probes a gap in social cognition capabilities of multimodal LLMs, with implications for understanding emergent group behavior.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9680814c9d1989b4a3c300d553df134a3700805fa2927ed01bc46dfaacb3f9f7"},"motivation":"True general intelligence requires not only a model of the physical world but also a social world model: the capacity to infer how individual mental states interact and crystallize into group-level outcomes.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04184","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_finstressts_aacbcf42","familyId":"bmf_7cb0bbf0767d","name":"FinStressTS","oneLine":"FinStressTS is a synthetic benchmark for financial time-series forecasting with controllable mechanisms and diagnostic environments.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["q-fin.CP"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03184","pdf":"https://arxiv.org/pdf/2606.03184","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03184"},"evidence":{"snippet":"We introduce FinStressTS, a mechanism-aware synthetic benchmark that links model behavior to controlled structural causes.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03184"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FinStressTS is a synthetic benchmark for financial time-series forecasting with controllable mechanisms and diagnostic environments.","whyItMatters":"It enables failure attribution in forecasting models, improving understanding of performance under different data-generating processes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"13745d96f254df37e0763352802185611a59db530ba4e9a71d9659f0b2a5c902"},"motivation":"Financial forecasting is difficult due to low signal-to-noise ratios, latent factors, heavy tails, regime shifts, and jumps.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03184","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_evid-bench_e9fc3b86","familyId":"bmf_79757df46833","name":"EVID-Bench","oneLine":"EVID-Bench is a benchmark for search-grounded video misinformation detection, covering 222 videos and multiple manipulation types.","area":"Vision & 3D","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04098","pdf":"https://arxiv.org/pdf/2606.04098","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04098"},"evidence":{"snippet":"We introduce \\textbf{EVID-Bench}, a benchmark for search-grounded video misinformation detection, where a system must search the open web for related videos and identify what information is false through cross-video comparison.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04098"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"EVID-Bench is a benchmark for search-grounded video misinformation detection, covering 222 videos and multiple manipulation types.","whyItMatters":"It highlights challenges in detecting video misinformation that requires external evidence, informing verification systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ef30a0c3b20ebc3419c8b670dcb196c5ddfad62ca2662af0b98b958a5b5e9822"},"motivation":"Video misinformation increasingly operates at the semantic and evidential level: authentic footage may be selectively edited, temporally reordered, spliced across sources, or augmented with AI-generated content to construct false narratives.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04098","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_entsql_088384b0","familyId":"bmf_c6be492ed41d","name":"EntSQL","oneLine":"EntSQL evaluates text-to-SQL systems on enterprise knowledge grounding, with 1,066 Chinese-English examples across five business domains requiring private business knowledge. Systems generate SQL from questions and schema, with some provided long-form documents.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["Long Context"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03363","pdf":"https://arxiv.org/pdf/2606.03363","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03363"},"evidence":{"snippet":"We introduce EntSQL, an enterprise-oriented Text-to-SQL benchmark for evaluating long-context grounding over proprietary business documents.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03363"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EntSQL evaluates text-to-SQL systems on enterprise knowledge grounding, with 1,066 Chinese-English examples across five business domains requiring private business knowledge. Systems generate SQL from questions and schema, with some provided long-form documents.","whyItMatters":"Existing text-to-SQL benchmarks overlook enterprise scenarios where SQL generation depends on proprietary business knowledge. EntSQL measures the ability to ground SQL generation in long-context enterprise documents, revealing a significant performance gap in current systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"18604bf22b7018a140fdaf137b507c36ec00c009dd6335570dac90be6dc09a21"},"motivation":"Text-to-SQL enables natural language access to databases, and recent LLMs have substantially advanced its capabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03363","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Long Context & Memory"],"domainScope":"general"},{"id":"bm_emem-bench_594a2f57","familyId":"bmf_3800709e05be","name":"eMEM-Bench","oneLine":"eMEM-Bench v1 evaluates embodied memory systems through 988 probes across eight cognitive-psychology paradigms (DRM lures, pattern separation, pattern completion, source monitoring, context-dependent retrieval, long-horizon interference, serial position, foil retention curve) in ProcTHOR-10K scenes.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.RO"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03374","pdf":"https://arxiv.org/pdf/2606.03374","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03374"},"evidence":{"snippet":"In addition we introduce eMEM-Bench v1, a benchmark we construct over ProcTHOR-10K scenes for embodied memory evaluation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03374"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"eMEM-Bench v1 evaluates embodied memory systems through 988 probes across eight cognitive-psychology paradigms (DRM lures, pattern separation, pattern completion, source monitoring, context-dependent retrieval, long-horizon interference, serial position, foil retention curve) in ProcTHOR-10K scenes.","whyItMatters":"Existing agent memory benchmarks lack diagnostic depth. eMEM-Bench provides interpretable results tied to memory-systems literature, allowing fine-grained comparison of embodied memory architectures beyond surface-task accuracy.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"80342a0435f2305b58474c90aaa881bd9bd39666b1bacf0636861f944d0e4c84"},"motivation":"We present eMEM (Embodied Memory), a hybrid graph-based memory system for embodied agents operating in physical environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03374","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_dosebench_cf8cdec2","familyId":"bmf_828ba2bfddf1","name":"DOSEBENCH","oneLine":"DOSEBENCH evaluates LLM decision-making on over-the-counter dosing questions, with 81 curated scenarios for adult acetaminophen and ibuprofen use. Correct answers require tracking dose timing, computing rolling 24-hour intake, and following product-label constraints.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.04262","pdf":"https://arxiv.org/pdf/2606.04262","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04262"},"evidence":{"snippet":"We introduce DOSEBENCH, a focused benchmark of 81 curated OTC dosing scenarios focused on adult acetaminophen and ibuprofen use, with manually annotated gold references.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04262"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DOSEBENCH evaluates LLM decision-making on over-the-counter dosing questions, with 81 curated scenarios for adult acetaminophen and ibuprofen use. Correct answers require tracking dose timing, computing rolling 24-hour intake, and following product-label constraints.","whyItMatters":"Addresses an underexplored safety-relevant medical QA setting. Evaluates temporal reasoning, constraint following, and uncertainty handling, showing that confident responses can violate dosing constraints.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"35cb50ac28ed5751da38f6dc540086da39e58e9b4641d5d407186162205a041d"},"motivation":"Large language models (LLMs) are increasingly used for everyday health questions, including whether a user can safely take another dose of an over-the-counter (OTC) medication.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04262","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_deskcraft_5fe792c2","familyId":"bmf_eaf217e35324","name":"DeskCraft","oneLine":"DeskCraft evaluates desktop GUI agents on long-horizon professional workflows and human-in-the-loop collaboration across 538 executable tasks in live Ubuntu desktop environments. Covers design, video, audio, and 3D creation software, with execution-based verification.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03103","pdf":"https://arxiv.org/pdf/2606.03103","project":null,"code":"https://github.com/mrwwk/DeskCraft","data":null,"hfPaper":"https://huggingface.co/papers/2606.03103"},"evidence":{"snippet":"To address this issue, we introduce DeskCraft, a desktop GUI benchmark targeting long horizon creative and engineering workflows and proactive human-agent collaboration.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":null,"githubStars":90,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03103"},"ranking":{"90d":{"score":75,"rank":46,"coverage":0.7,"confidence":"Medium"}},"description":"DeskCraft evaluates desktop GUI agents on long-horizon professional workflows and human-in-the-loop collaboration across 538 executable tasks in live Ubuntu desktop environments. Covers design, video, audio, and 3D creation software, with execution-based verification.","whyItMatters":"Existing desktop benchmarks simplify tasks and lack human-agent interaction. DeskCraft measures agent performance on realistic workflows and proactive collaboration, identifying gaps in long-horizon delivery and clarification.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9197fd4bd53ae02283a924b4d2474a32ca619a045171b712ab56fadefa7887db"},"motivation":"Real-world professional desktop workflows in specialized creative and engineering software unfold over long horizons and often require human-in-the-loop coordination, where agents proactively seek necessary information and users provide additional instructions, clarifications, feedback, or corrections as the task progresses.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03103","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DeskCraft team","organizationType":"community","sourceUrl":"https://github.com/mrwwk/DeskCraft","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_delegateci-bench_0d61203c","familyId":"bmf_3a846b227d7c","name":"DelegateCI-Bench","oneLine":"DelegateCI-Bench evaluates privacy-conscious query rewriting for LLM delegation, with 3,167 samples combining synthetic data across 20 task types, real user queries from WildChat, and a medical challenge set. Systems rewrite queries to suppress non-essential sensitive information.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CR"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04067","pdf":"https://arxiv.org/pdf/2606.04067","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04067"},"evidence":{"snippet":"We introduce DelegateCI-Bench, the first task based Contextual Integrity benchmark for privacy-conscious delegation, comprising 3,167 samples that combine high quality synthetic data spanning 11 tasks and 20 task types, WildChat based real user queries, and a medical challenge set with dense sensitive information.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04067"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DelegateCI-Bench evaluates privacy-conscious query rewriting for LLM delegation, with 3,167 samples combining synthetic data across 20 task types, real user queries from WildChat, and a medical challenge set. Systems rewrite queries to suppress non-essential sensitive information.","whyItMatters":"Addresses a gap in privacy benchmarks by focusing on task-based necessity rather than type-based PII redaction. Measures privacy-utility tradeoff in delegation, supporting safer LLM use.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"96263337428be6b832d99ca1612b8c63f7071289fac80f14187161d10e542d53"},"motivation":"As LLMs become increasingly woven into everyday workflows, user queries sent to cloud hosted LLMs routinely mix task-essential content with task non-essential sensitive disclosures, yet type based PII redaction is context agnostic and may raise two issues: over disclosing untyped sensitive context and over removing answer bearing spans.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04067","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_clinicalmc_22427e9b","familyId":"bmf_297bec9ba523","name":"ClinicalMC","oneLine":"ClinicalMC evaluates LLM clinical decision-making across multi-course patient trajectories, with 1,275 Chinese and 5,804 English samples spanning four stages from admission to discharge. Includes triage, examination, diagnosis, treatment, and final diagnosis.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03157","pdf":"https://arxiv.org/pdf/2606.03157","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03157"},"evidence":{"snippet":"To address this gap, we propose ClinicalMC, a benchmark for multi-course clinical decision-making.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03157"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ClinicalMC evaluates LLM clinical decision-making across multi-course patient trajectories, with 1,275 Chinese and 5,804 English samples spanning four stages from admission to discharge. Includes triage, examination, diagnosis, treatment, and final diagnosis.","whyItMatters":"Existing clinical benchmarks focus on single-course scenarios, missing the complexity of evolving patient conditions. ClinicalMC enables evaluation of dynamic multi-turn decision-making, supporting safer LLM deployment in healthcare.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6376e4263cd7f0f2ac1d5cfd8a45f693e5bf8729c2a91823ffa73bfd38025e94"},"motivation":"Large language models (LLMs) have been widely adopted in healthcare, yet they still encounter significant challenges in complex clinical decision-making scenarios.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03157","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_chemcotbench-v2_5360db80","familyId":"bmf_43f17e5151db","name":"ChemCoTBench-V2","oneLine":"ChemCoTBench-V2 evaluates chemical reasoning at the process level, with 5,620 samples across molecular understanding, editing, optimization, and reaction prediction. It checks intermediate steps using deterministic chemistry rules and reference traces.","area":"Language & Knowledge","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03660","pdf":"https://arxiv.org/pdf/2606.03660","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03660"},"evidence":{"snippet":"We introduce ChemCoTBench-V2, a rule-verifiable diagnostic benchmark for low-cost, auditable evaluation of structured, verifier-addressable chemical reasoning traces.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03660"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"ChemCoTBench-V2 evaluates chemical reasoning at the process level, with 5,620 samples across molecular understanding, editing, optimization, and reaction prediction. It checks intermediate steps using deterministic chemistry rules and reference traces.","whyItMatters":"Existing chemistry benchmarks only score final answers, missing violations in reasoning. ChemCoTBench-V2 provides auditable, verifiable process-level evaluation with three separate signals, enabling fine-grained model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c3e8746331d8b2fba17760cb5543ca5052e315212ec565b0c995d37dce291805"},"motivation":"Large language models are increasingly used as chemistry assistants, yet most chemistry benchmarks still score only final answers.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03660","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_bigfinancebench_c8640763","familyId":"bmf_4b9b21faaf9f","name":"BigFinanceBench","oneLine":"BigFinanceBench evaluates financial-research agents on open-ended tasks with 928 items, each paired with a ground-truth answer and a point-weighted rubric decomposing the derivation into steps. Supports partial-credit scoring across 36,241 rubric points.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-02","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.03829","pdf":"https://arxiv.org/pdf/2606.03829","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.03829"},"evidence":{"snippet":"We introduce BigFinanceBench, a 928-item expert-authored benchmark of open-ended financial-research tasks in which each item pairs a ground-truth reference answer with a point-weighted rubric that decomposes the derivation into independently checkable steps.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.03829"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"BigFinanceBench evaluates financial-research agents on open-ended tasks with 928 items, each paired with a ground-truth answer and a point-weighted rubric decomposing the derivation into steps. Supports partial-credit scoring across 36,241 rubric points.","whyItMatters":"Existing finance benchmarks evaluate subskills or final answers, not the auditable derivation. BigFinanceBench measures workflow quality, allowing localization of failures and better assessment of decision-relevant outputs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"883397bd70250ab65f8b39beec3514a8acdf14a941036f77a5cfae507da703d9"},"motivation":"Financial-research answers are decision-relevant only when another analyst can audit how they were produced: which source was chosen, which period and accounting definition were used, which assumptions were made, and how the calculation was performed.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.03829","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_x-stream_b79ec653","familyId":"bmf_229d06cf6ea7","name":"X-Stream","oneLine":"Evaluates multimodal large language models on multi-stream streaming understanding, with 4,220 QA pairs across 932 videos covering 11 subtasks in multi-window, multi-view, and multi-device scenarios, using a dual-verification construction pipeline and online inference under a fixed average video-token rate.","area":"Vision & 3D","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Automotive"],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02482","pdf":"https://arxiv.org/pdf/2606.02482","project":"https://peiwensun2000.github.io/xstream/","code":"https://github.com/PeiwenSun2000/X-Stream","data":null,"hfPaper":"https://huggingface.co/papers/2606.02482"},"evidence":{"snippet":"To bridge this, we introduce X-Stream, the first benchmark dedicated to multi-stream streaming understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":36,"hfDailySubmittedAt":"2026-06-02T00:00:00.000Z","githubStars":34,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02482"},"ranking":{"90d":{"score":73,"rank":56,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates multimodal large language models on multi-stream streaming understanding, with 4,220 QA pairs across 932 videos covering 11 subtasks in multi-window, multi-view, and multi-device scenarios, using a dual-verification construction pipeline and online inference under a fixed average video-token rate.","whyItMatters":"Existing benchmarks focus on single-stream video understanding, leaving a gap for evaluating concurrent video streams common in live sports, autonomous driving, and multi-screen applications. This benchmark provides a practical evaluation protocol for multi-stream reasoning and exposes limitations in current models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b1ff0c2b2e664aee9dd1a89b560d911f0249840395675cd01ee014873fe5bb87"},"motivation":"While video streaming understanding has made significant strides, real-world applications, such as live sports broadcasting, autonomous driving, and multi-screen collaboration, inherently demand continuous, multi-stream interactions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02482","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MMLab, CUHK","organizationType":"academic-lab","sourceUrl":"https://github.com/PeiwenSun2000/X-Stream","role":"benchmark-publisher"},{"name":"Huawei Inc.","organizationType":"company-research-lab","sourceUrl":"https://github.com/PeiwenSun2000/X-Stream","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_worldcoder-bench_8f868de5","familyId":"bmf_46b10b77b482","name":"WorldCoder-Bench","oneLine":"Evaluates autonomous physically grounded 3D world synthesis from natural language, with 2,026 expert-curated tasks across Simulation, Rendering, and Application scenarios, using execution-based verification via StateProbe to check hidden behavioral contracts over runtime states.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01869","pdf":"https://arxiv.org/pdf/2606.01869","project":"https://anonymous.4open.science/r/WorldCoder-Bench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01869"},"evidence":{"snippet":"We introduce WorldCoder-Bench, a benchmark for autonomous, physically grounded 3D world synthesis.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01869"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Evaluates autonomous physically grounded 3D world synthesis from natural language, with 2,026 expert-curated tasks across Simulation, Rendering, and Application scenarios, using execution-based verification via StateProbe to check hidden behavioral contracts over runtime states.","whyItMatters":"Existing web-generation benchmarks observe only pixels or DOM nodes, missing the mechanics of Three.js worlds inside canvas elements. This benchmark provides a protocol for verifying hidden contracts, enabling assessment of correctness-adjusted cost and time efficiency for 3D world synthesis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3227ed109d8f98090ac73e1e2cd973521870dd9ffe97c2ac2216641c779ad4bb"},"motivation":"Large language models (LLMs) are increasingly asked not only to write static interfaces, but to construct executable interactive worlds from natural language.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01869","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"WorldCoder-Bench team","organizationType":"academic-lab","sourceUrl":"https://anonymous.4open.science/r/WorldCoder-Bench/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_the-image-reconstruction-game_055fc5b9","familyId":"bmf_33f45be8b058","name":"The Image Reconstruction Game","oneLine":"A benchmark for iterative multimodal dialogue where a vision-language model issues corrective instructions to an image generator over multiple turns, with accumulated common ground observable as a rendered image.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01901","pdf":"https://arxiv.org/pdf/2606.01901","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01901"},"evidence":{"snippet":"We introduce the Image Reconstruction Game, a fully automated benchmark in which a vision-language model issues corrective instructions to an image generator across multiple turns, making accumulated common ground directly observable as a rendered image.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01901"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark for iterative multimodal dialogue where a vision-language model issues corrective instructions to an image generator over multiple turns, with accumulated common ground observable as a rendered image.","whyItMatters":"Addresses the evaluation of interactive language-vision systems and common ground building, but lacks a clear public comparison path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0dde71fac170cc5190020b94b0e20a3363f009c360739789a157fe9b9c2bf3ca"},"motivation":"We introduce the Image Reconstruction Game, a fully automated benchmark in which a vision-language model issues corrective instructions to an image generator across multiple turns, making accumulated common ground directly observable as a rendered image.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01901","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_telbench_d23c11db","familyId":"bmf_8c4c87f14d99","name":"TELBench","oneLine":"Evaluates span-level error localization in deep-research agent trajectories. TELBench comprises 1,000 instances with annotations of harmful error spans among normal exploration, failed searches, tentative hypotheses, and harmless noise, scored by span-level localization and first-error accuracy.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02060","pdf":"https://arxiv.org/pdf/2606.02060","project":null,"code":"https://github.com/NJU-LINK/DRIFT","data":null,"hfPaper":"https://huggingface.co/papers/2606.02060"},"evidence":{"snippet":"From these annotations, we build TELBench, a 1,000-instance benchmark for identifying error spans among normal exploration, failed searches, tentative hypotheses, and harmless noise.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":58,"hfDailySubmittedAt":"2026-06-04T00:00:00.000Z","githubStars":22,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02060"},"ranking":{"90d":{"score":71,"rank":69,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates span-level error localization in deep-research agent trajectories. TELBench comprises 1,000 instances with annotations of harmful error spans among normal exploration, failed searches, tentative hypotheses, and harmless noise, scored by span-level localization and first-error accuracy.","whyItMatters":"Final-answer evaluation does not reveal which trajectory steps make answers unreliable. This benchmark enables process-level reliability assessment and comparison of error localization methods for deep-research agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ad0404261eeff9c40b323be9628c5f11c01515f0877b2df8dc1b3200e78aa8fb"},"motivation":"Deep-research agents solve tasks through long trajectories of search, tool use, evidence inspection, and answer synthesis.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02060","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"NJU-LINK","organizationType":"academic-lab","sourceUrl":"https://github.com/NJU-LINK/DRIFT","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_spade-bench_b8164972","familyId":"bmf_96fc99f9be33","name":"SPADE-Bench","oneLine":"Benchmark for evaluating spontaneous plan-action divergence in agents, integrating actual tool execution and controlled pressure scenarios to distinguish strategic deception from hallucination.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.02380","pdf":"https://arxiv.org/pdf/2606.02380","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02380"},"evidence":{"snippet":"To assess this, we introduce SPADE-Bench, a benchmark designed to evaluate spontaneous plan-action divergence.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02380"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Benchmark for evaluating spontaneous plan-action divergence in agents, integrating actual tool execution and controlled pressure scenarios to distinguish strategic deception from hallucination.","whyItMatters":"Addresses the critical risk of agent deception in autonomous systems, but lacks a public comparison path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3fbc72802ddfa61546f9fcfd3295107dc956b1ef8e312a1502d72f7d7a6ec9b9"},"motivation":"As LLM-based agents expand their operational scope, reliability becomes a prerequisite for real-world deployment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02380","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_smh-bench_ef4613ad","familyId":"bmf_eb0612cdd00c","name":"SMH-Bench","oneLine":"Benchmark for LLM agents in smart-home environments, with 1,100 tasks across 7 categories and 22 subcategories, built on the executable HomeEnv simulator, stratified by home complexity.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01912","pdf":"https://arxiv.org/pdf/2606.01912","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01912"},"evidence":{"snippet":"To address these limitations, we introduce SMH-Bench, a comprehensive benchmark for evaluating LLMs in smart-home environments.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01912"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Benchmark for LLM agents in smart-home environments, with 1,100 tasks across 7 categories and 22 subcategories, built on the executable HomeEnv simulator, stratified by home complexity.","whyItMatters":"Addresses deficiencies in existing smart-home benchmarks by evaluating state-dependent reasoning across devices, but lacks a public comparison path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"29c5a612b2da0a8f544596988e46f44d3b446c5c05c52d96f065c4d053529a2c"},"motivation":"Smart homes are evolving toward complex state-dependent living environments, requiring Large Language Models (LLMs) to reason over user intent, preferences, and multi-device interactions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01912","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_skillharm_560fd4c5","familyId":"bmf_8422ad323405","name":"SkillHarm","oneLine":"Benchmark of skill-based attacks across the skill-use lifecycle, with 879 attack samples across 71 skills, evaluating Fixed-Payload Poisoning and Self-Mutating Poisoning scenarios across 12 risk types, with attack success rate as primary metric.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02540","pdf":"https://arxiv.org/pdf/2606.02540","project":null,"code":"https://github.com/OSU-NLP-Group/SkillHarm","data":null,"hfPaper":"https://huggingface.co/papers/2606.02540"},"evidence":{"snippet":"To bridge these gaps, we introduce SkillHarm, a benchmark of skill-based attacks across the skill-use lifecycle, paired with a systematic taxonomy of skill-relevant risks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":10,"hfDailySubmittedAt":"2026-06-10T00:00:00.000Z","githubStars":10,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02540"},"ranking":{"90d":{"score":62,"rank":122,"coverage":0.7,"confidence":"Medium"}},"description":"Benchmark of skill-based attacks across the skill-use lifecycle, with 879 attack samples across 71 skills, evaluating Fixed-Payload Poisoning and Self-Mutating Poisoning scenarios across 12 risk types, with attack success rate as primary metric.","whyItMatters":"Existing studies evaluate poisoned skills within single task executions and use ad-hoc risk lists. This benchmark systematically covers lifecycle-aware attacks and provides a taxonomy and construction pipeline for reproducible evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d9bf68d429d90f900361bb66d90fcee3ebc365ed25421becd42c88b9fb1e5dc7"},"motivation":"Agent skills occupy a privileged position in the agent workflow, as agents are expected to implicitly follow and execute them, rendering third-party skills a vulnerable attack surface.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02540","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"OSU-NLP-Group","organizationType":"academic-lab","sourceUrl":"https://github.com/OSU-NLP-Group/SkillHarm","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_rotoutbench_a7d65303","familyId":"bmf_67067fe3674d","name":"RotOutBench","oneLine":"Paired diagnostic benchmark for rotated-outcome prediction in vision-language models, spanning open visual cases and controlled text-image rotations, with accuracy metrics for direct reading and prediction.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.07641","pdf":"https://arxiv.org/pdf/2606.07641","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07641"},"evidence":{"snippet":"To isolate this gap, we introduce RotOutBench, a paired diagnostic benchmark spanning open visual cases and controlled text-image rotations.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07641"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Paired diagnostic benchmark for rotated-outcome prediction in vision-language models, spanning open visual cases and controlled text-image rotations, with accuracy metrics for direct reading and prediction.","whyItMatters":"Evaluates a specific cognitive ability in VLMs, but lacks a standalone public comparison path.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"621eed638574e244809bd38bf77a7430150c1d406be542f2ddae745a35773eed"},"motivation":"Can vision-language models predict what a 180{\\deg} rotation would reveal from the original image alone?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07641","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_rolecde_7c0f211a","familyId":"bmf_ce6bd9dfe8c8","name":"RoleCDE","oneLine":"RoleCDE evaluates role-playing agents under structured conflicts between role-specific values and alignment-oriented constraints. It comprises approximately 8,000 role profiles and 240,000 dilemma instances across three difficulty levels and eight role categories, with scoring via LLM-as-a-judge.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01552","pdf":"https://arxiv.org/pdf/2606.01552","project":null,"code":"https://github.com/rabbitrose/RoleCDE","data":null,"hfPaper":"https://huggingface.co/papers/2606.01552"},"evidence":{"snippet":"To address this gap, we introduce RoleCDE, the first benchmark designed to evaluate RPAs under structured conflicts between role-specific values and alignment-oriented constraints.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01552"},"ranking":{"90d":{"score":39,"rank":298,"coverage":0.55,"confidence":"Low"}},"description":"RoleCDE evaluates role-playing agents under structured conflicts between role-specific values and alignment-oriented constraints. It comprises approximately 8,000 role profiles and 240,000 dilemma instances across three difficulty levels and eight role categories, with scoring via LLM-as-a-judge.","whyItMatters":"Existing benchmarks focus on surface fidelity and lack coverage of decision-making under role-alignment value conflicts. RoleCDE provides a systematic evaluation of how agents resolve such conflicts, revealing systematic biases and offering a tool for improving alignment and role consistency.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c20da1b38ee79974202a55eca82e8b9607810ea76939e9cab77f375de94bd6df"},"motivation":"Role-playing agents(RPAs) are widely used to steer large language models(LLMs) toward role-consistent behavior, yet existing benchmarks mainly evaluate surface-level fidelity and offer limited insight into decision making under role-alignment value conflicts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01552","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"RoleCDE Team","organizationType":"academic-lab","sourceUrl":"https://github.com/rabbitrose/RoleCDE","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_robotrustbench_454946c5","familyId":"bmf_2309dd08a194","name":"RoboTrustBench","oneLine":"RoboTrustBench evaluates the trustworthiness of video world models for robotic manipulation across four scenarios: Normal, Constraint-Sensitive, Counterfactual, and Adversarial. It contains 1,207 expert-validated instruction-image pairs from DROID episodes and a six-dimensional evaluation protocol with 13 fine-grained criteria.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01600","pdf":"https://arxiv.org/pdf/2606.01600","project":"https://huiqiongli.github.io/RoboTrustBench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01600"},"evidence":{"snippet":"We introduce RoboTrustBench, a benchmark for evaluating the trustworthiness of video world models under four scenarios: Normal, Constraint-Sensitive, Counterfactual, and Adversarial.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01600"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"RoboTrustBench evaluates the trustworthiness of video world models for robotic manipulation across four scenarios: Normal, Constraint-Sensitive, Counterfactual, and Adversarial. It contains 1,207 expert-validated instruction-image pairs from DROID episodes and a six-dimensional evaluation protocol with 13 fine-grained criteria.","whyItMatters":"Existing benchmarks for video world models largely overlook trustworthiness aspects such as constraint reasoning, counterfactual grounding, and safety. RoboTrustBench provides a structured evaluation to assess these capabilities, offering practical guidance for selecting and improving models for safe robotic manipulation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"15548aa1d2295b999b236d057b81f446c1a22b8293446670ef77579a87a37f08"},"motivation":"Video world models are increasingly used in robotic manipulation, yet existing benchmarks mostly evaluate them under valid, feasible, and safe instructions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01600","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"RoboTrustBench Team","organizationType":"academic-lab","sourceUrl":"https://huiqiongli.github.io/RoboTrustBench/","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_robosemanticbench_9bca3113","familyId":"bmf_85bece8521c2","name":"RoboSemanticBench","oneLine":"RoboSemanticBench (RSB) is an embodied benchmark evaluating whether vision-language-action models use instruction semantics to select and grasp the correct physical target among candidate blocks in response to math or general-knowledge questions. It includes six suites with four- or ten-choice variants, procedural arithmetic, GSM8K-style, and MMLU-style questions, with diagnostic metrics separating task success from grasp success.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.RO"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02277","pdf":"https://arxiv.org/pdf/2606.02277","project":null,"code":"https://github.com/ZGC-EmbodyAI/RoboSemanticBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.02277"},"evidence":{"snippet":"We introduce RoboSemanticBench (RSB), an embodied benchmark for diagnosing semantic grounding in action prediction: whether post-trained VLA models can use complex instruction semantics to select and manipulate the correct physical target.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":7,"hfDailySubmittedAt":"2026-06-02T00:00:00.000Z","githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02277"},"ranking":{"90d":{"score":52,"rank":183,"coverage":0.7,"confidence":"Medium"}},"description":"RoboSemanticBench (RSB) is an embodied benchmark evaluating whether vision-language-action models use instruction semantics to select and grasp the correct physical target among candidate blocks in response to math or general-knowledge questions. It includes six suites with four- or ten-choice variants, procedural arithmetic, GSM8K-style, and MMLU-style questions, with diagnostic metrics separating task success from grasp success.","whyItMatters":"RSB addresses the evaluation gap of measuring whether VLA models actually ground instruction semantics in action prediction, separating low-level manipulation from semantic selection. It provides a controlled, repeatable protocol with held-out questions, useful for diagnosing model capabilities and guiding improvements in embodied semantic understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"587e7e6271eb53d0d9a42c28e6a342f95702b75866ea73cb01aba2d2069c3b3b"},"motivation":"Vision-language-action (VLA) models are built on the premise that semantic understanding from pretrained language or vision-language backbones should guide robot action prediction.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02277","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ZGC-EmbodyAI","organizationType":"academic-lab","sourceUrl":"https://github.com/ZGC-EmbodyAI/RoboSemanticBench","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_rescuebench_f2263257","familyId":"bmf_6b6323c23009","name":"RescueBench","oneLine":"RescueBench evaluates embodied search-and-rescue agents in simulated photo-realistic environments. It comprises a four-stage pipeline: multimodal exploration, target rescue, memory-guided return, and final handoff. Five difficulty levels vary environmental complexity, clue ambiguity, and spatial hierarchy. Automatic episode generation and annotation support scalable evaluation. A unified benchmark framework and runner scripts provide standardized scoring.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01848","pdf":"https://arxiv.org/pdf/2606.01848","project":null,"code":"https://github.com/wukui-muc/RescueBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.01848"},"evidence":{"snippet":"We introduce RescueBench, a photo-realistic diagnostic benchmark that instantiates SAR as a four-stage pipeline: multimodal exploration, target rescue, memory-guided return, and final handoff.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01848"},"ranking":{"90d":{"score":51,"rank":188,"coverage":0.55,"confidence":"Low"}},"description":"RescueBench evaluates embodied search-and-rescue agents in simulated photo-realistic environments. It comprises a four-stage pipeline: multimodal exploration, target rescue, memory-guided return, and final handoff. Five difficulty levels vary environmental complexity, clue ambiguity, and spatial hierarchy. Automatic episode generation and annotation support scalable evaluation. A unified benchmark framework and runner scripts provide standardized scoring.","whyItMatters":"Search-and-rescue benchmarks typically test capabilities in isolation. RescueBench addresses the gap of composite workflows where failures may compound across stages. It provides stage-level diagnostics to identify bottlenecks (e.g., exploration, memory) separately from end-to-end performance, helping practitioners target improvements in embodied agents for realistic rescue tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"64a76691b682d95054288a0a3cf96cc73246b329fb3ee052d76fc4361bf07ea3"},"motivation":"Search-and-rescue (SAR) requires embodied agents to explore unfamiliar environments under multimodal uncertainty, perform multi-stage interactions, and retrieve spatial memory over long horizons.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01848","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_rescast-100k_da1d15f3","familyId":"bmf_4fbc0d9dc8e9","name":"RESCAST-100K","oneLine":"RESCAST-100K evaluates cross-domain residential load and indoor temperature forecasting. It provides ~100,000 EnergyPlus-simulated U.S. homes with 15-minute time series for total load, HVAC load, and indoor temperature, plus weather, setpoints, and static covariates. It includes configurable domain axes and integrates five real-world datasets for sim-to-real evaluation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02852","pdf":"https://arxiv.org/pdf/2606.02852","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02852"},"evidence":{"snippet":"We introduce RESCAST-100K, a large-scale residential forecasting benchmark for studying cross-domain generalization.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02852"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RESCAST-100K evaluates cross-domain residential load and indoor temperature forecasting. It provides ~100,000 EnergyPlus-simulated U.S. homes with 15-minute time series for total load, HVAC load, and indoor temperature, plus weather, setpoints, and static covariates. It includes configurable domain axes and integrates five real-world datasets for sim-to-real evaluation.","whyItMatters":"Existing residential forecasting datasets are narrow and lack structured cross-domain evaluation. RESCAST-100K enables systematic assessment of transfer learning and domain adaptation under controlled shifts, supporting better generalization in home energy management and grid-scale applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8e337ce9dffe970f3cefac4f15113cbbf5cc0b410b2f0bd60a079ba229d41361"},"motivation":"Accurate short-term forecasting of residential energy load and indoor temperature is essential for home energy management systems, grid-level demand response, and community energy efficiency efforts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02852","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_plan2map_b8494f1d","familyId":"bmf_133ef96ac8ba","name":"Plan2Map","oneLine":"Plan2Map evaluates document-grounded geospatial boundary reconstruction from UK planning records. Systems input a planning document and output a GeoJSON boundary, scored against held-out reference boundaries via IoU.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning"],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02747","pdf":"https://arxiv.org/pdf/2606.02747","project":"https://odeb1.github.io/Plan2Map_Project_Page/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02747"},"evidence":{"snippet":"We introduce Plan2Map, a 208-case multimodal benchmark for document-grounded geospatial boundary reconstruction from UK planning records.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02747"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Plan2Map evaluates document-grounded geospatial boundary reconstruction from UK planning records. Systems input a planning document and output a GeoJSON boundary, scored against held-out reference boundaries via IoU.","whyItMatters":"Plan2Map addresses the gap in evaluating multimodal geospatial reconstruction from public planning documents, providing a concrete testbed with held-out scoring for comparing systems on a complex, real-world task.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"09e8bc7061dbfe64ade1b66c56382c2c095ae5aff0cb41b70498d2bd7946357c"},"motivation":"Planning records define restrictions over geographic areas, but their source documents often provide only indirect spatial evidence rather than machine-readable boundaries.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02747","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_pasbench-video_ab3b3eb4","familyId":"bmf_6df4c19f80cb","name":"PaSBench-Video","oneLine":"PaSBench-Video is a 740-video benchmark for proactive safety warning with 481 risk and 259 no-risk videos across driving, healthcare, daily life, and industrial production. Annotations include frame-level risk onset and accident boundaries. Models must process video causally and output a warning that is temporally calibrated and content-correct.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02443","pdf":"https://arxiv.org/pdf/2606.02443","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02443"},"evidence":{"snippet":"We present PaSBench-Video, a 740-video benchmark with 481 risk and 259 no-risk videos across four domains: driving, healthcare, daily life, and industrial production.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02443"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PaSBench-Video is a 740-video benchmark for proactive safety warning with 481 risk and 259 no-risk videos across driving, healthcare, daily life, and industrial production. Annotations include frame-level risk onset and accident boundaries. Models must process video causally and output a warning that is temporally calibrated and content-correct.","whyItMatters":"This benchmark addresses a gap in evaluating video MLLMs for real-time safety monitoring, emphasizing temporal calibration and false-positive control on safe scenes. It provides a standardized protocol to compare models' ability to issue timely warnings, which is critical for deployment in safety-sensitive applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8c790c25e946c2839989c8470d79faece82637cd7edcdd4911882a0fce6b870a"},"motivation":"Between the first visible sign of danger and the moment an accident occurs, there is often a window where intervention remains possible.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02443","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_moment-video_26d94dc0","familyId":"bmf_3b12aecab48e","name":"Moment-Video","oneLine":"Moment-Video evaluates video MLLMs on momentary visual event understanding through 1,000 human-verified video-QA pairs across four task types: temporal occurrence, temporal counting, action description, and temporal reasoning.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02522","pdf":"https://arxiv.org/pdf/2606.02522","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02522"},"evidence":{"snippet":"We introduce Moment-Video, a benchmark for diagnosing the temporal fidelity of video MLLMs through momentary visual event understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":13,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02522"},"ranking":{"90d":{"score":54,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Moment-Video evaluates video MLLMs on momentary visual event understanding through 1,000 human-verified video-QA pairs across four task types: temporal occurrence, temporal counting, action description, and temporal reasoning.","whyItMatters":"It addresses the gap in evaluating models' ability to capture brief answer-critical visual evidence, which is common in practical video understanding and not covered by general video benchmarks. The results show significant room for improvement, offering a diagnostic tool for model development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"de72bb5d2df58b3152d36085960cec92ec1d9654f413e113c8033b53390744aa"},"motivation":"Video multimodal large language models (MLLMs) have made rapid progress on general and long-form video understanding, yet their ability to preserve brief answer-critical visual evidence remains underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02522","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mcp-persona_e4dd478c","familyId":"bmf_d0faf0a6b15e","name":"MCP-Persona","oneLine":"MCP-Persona evaluates LLM agents on real-world personalized MCP tools across social media, collaboration, email, and content management applications. It includes 173 tool-chain tasks, 139 unique tools, and 18 MCP servers, with a fully automated environment simulation pipeline for reproducible evaluation.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02470","pdf":"https://arxiv.org/pdf/2606.02470","project":null,"code":"https://github.com/wwh0411/MCP-Persona","data":null,"hfPaper":"https://huggingface.co/papers/2606.02470"},"evidence":{"snippet":"To bridge this critical gap, we introduce MCP-Persona, the first benchmark specifically designed for evaluating agent performance on real-world, personalized MCP tools.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":16,"hfDailySubmittedAt":"2026-06-02T00:00:00.000Z","githubStars":7,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02470"},"ranking":{"90d":{"score":59,"rank":138,"coverage":0.7,"confidence":"Medium"}},"description":"MCP-Persona evaluates LLM agents on real-world personalized MCP tools across social media, collaboration, email, and content management applications. It includes 173 tool-chain tasks, 139 unique tools, and 18 MCP servers, with a fully automated environment simulation pipeline for reproducible evaluation.","whyItMatters":"Existing benchmarks overlook personalized MCP tool use, which is critical for practical agent deployment. MCP-Persona provides a sandboxed, reproducible environment to test agents on realistic personal tasks, helping identify limitations and guiding development of more capable agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4521a2d9c11d1d10cc5db2c38447b4f940b0a15a163d1df64fa97ed9d8ab8741"},"motivation":"The Model Context Protocol (MCP) has emerged as a transformative standard for connecting large language models (LLMs) with external data sources and tools, and has been rapidly adopted across personal applications and development platforms.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02470","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MCP-Persona team","organizationType":"academic-lab","sourceUrl":"https://github.com/wwh0411/MCP-Persona","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_longjudgebench_1675bac5","familyId":"bmf_36c4ccde3d0e","name":"LongJudgeBench","oneLine":"LongJudgeBench evaluates LLM-as-a-judge performance on long-form outputs across six datasets covering pointwise, pairwise, and listwise protocols, with bilingual tasks and multiple prompt variants. It measures agreement with human judgments using accuracy, Spearman, and Kendall's tau.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01629","pdf":"https://arxiv.org/pdf/2606.01629","project":null,"code":"https://github.com/cjj826/LongJudgeBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.01629"},"evidence":{"snippet":"In this work, we introduce LongJudgeBench, a comprehensive benchmark for evaluating LLM judges on long-form outputs across diverse real-world scenarios and judging protocols.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01629"},"ranking":{"90d":{"score":51,"rank":189,"coverage":0.55,"confidence":"Low"}},"description":"LongJudgeBench evaluates LLM-as-a-judge performance on long-form outputs across six datasets covering pointwise, pairwise, and listwise protocols, with bilingual tasks and multiple prompt variants. It measures agreement with human judgments using accuracy, Spearman, and Kendall's tau.","whyItMatters":"Existing meta-evaluation benchmarks focus on short-form outputs, leaving a gap for long-form evaluation. This benchmark provides a standardized way to assess judge reliability across diverse scenarios, helping practitioners select or develop judges for long-form tasks where current models show instability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9c4ac4a6122fbef7436d7a2b26ecff815871da43423d65c19bf5b4396ded1e84"},"motivation":"As large language models (LLMs) are increasingly used for long-form generation, reliably evaluating long-form outputs has become a critical challenge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01629","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LongJudgeBench Team","organizationType":"academic-lab","sourceUrl":"https://github.com/cjj826/LongJudgeBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_ll-bench_df7573ac","familyId":"bmf_e383e91589a5","name":"LL-Bench","oneLine":"LL-Bench evaluates large-scale generative models on 16 low-level vision tasks using 2,469 real-world degraded images, with human preference and quality score annotations for model outputs.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02535","pdf":"https://arxiv.org/pdf/2606.02535","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02535"},"evidence":{"snippet":"To address this gap, we introduce \\textbf{LL-Bench}, a comprehensive \\textbf{Benchmark} for evaluating the capabilities of large-scale generative models on \\textbf{L}ow-\\textbf{L}evel vision tasks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02535"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"LL-Bench evaluates large-scale generative models on 16 low-level vision tasks using 2,469 real-world degraded images, with human preference and quality score annotations for model outputs.","whyItMatters":"Existing low-level vision benchmarks often focus on conventional models and lack alignment with human perception. LL-Bench provides a standardized evaluation suite to compare generative and restoration models on pixel-level tasks, supporting quality assessment and model selection.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7ba0d380642efecde1cd833658bd564758243f39f900f18ad4067939ea0eda2d"},"motivation":"Large-scale generative models have demonstrated remarkable capabilities across image generation and editing tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02535","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_k-browsecomp_c3715567","familyId":"bmf_7a5a02770094","name":"K-BrowseComp","oneLine":"K-BrowseComp evaluates web-browsing agents on 400 Korean-context problems, requiring multi-hop or parallel evidence retrieval from public Korean websites and returning a single short answer. Includes a 300-problem manually verified subset and a 100-problem synthetic stress-test split.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["Agents"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02404","pdf":"https://arxiv.org/pdf/2606.02404","project":null,"code":"https://github.com/prometheus-eval/K-BrowseComp","data":null,"hfPaper":"https://huggingface.co/papers/2606.02404"},"evidence":{"snippet":"We introduce K-BrowseComp, a web-browsing agent benchmark grounded in Korean contexts, consisting of 400 problems.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":59,"hfDailySubmittedAt":"2026-06-02T00:00:00.000Z","githubStars":14,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02404"},"ranking":{"90d":{"score":68,"rank":86,"coverage":0.7,"confidence":"Medium"}},"description":"K-BrowseComp evaluates web-browsing agents on 400 Korean-context problems, requiring multi-hop or parallel evidence retrieval from public Korean websites and returning a single short answer. Includes a 300-problem manually verified subset and a 100-problem synthetic stress-test split.","whyItMatters":"Existing agentic benchmarks overlook Korean-language browsing, and frontier models show a significant performance drop on this benchmark. It provides a public protocol for measuring Korean web navigation and evidence-tracking abilities, which is relevant for deploying agents in Korean-language settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5809d0ceaf4b49ab9433161381ed8445813f7968c9c9ca3a4c74fd5299c03617"},"motivation":"Frontier model evaluations are shifting from foundational capabilities (e.g., instruction following and reasoning) toward compositional, agentic ones, but Korean agentic benchmarks remain scarce.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02404","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"prometheus-eval","organizationType":"community","sourceUrl":"https://github.com/prometheus-eval/K-BrowseComp","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_hll_52c829e2","familyId":"bmf_f97ab7d41b89","name":"HLL","oneLine":"HLL is a benchmark that evaluates multimodal agents on interactive CAPTCHA verification in a closed-loop GUI environment, covering diverse task types such as text transcription, image selection, sliders, jigsaw puzzles, and logic-based challenges. Scoring is based on task completion and trace-conditioned validation.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02449","pdf":"https://arxiv.org/pdf/2606.02449","project":null,"code":"https://github.com/XinhaoS0101/HLL","data":null,"hfPaper":"https://huggingface.co/papers/2606.02449"},"evidence":{"snippet":"We introduce \\textbf{Humanity's Last Line of Verification (HLL)}, a controlled benchmark that uses interactive CAPTCHA verification to evaluate whether agents can cross this boundary through grounded, human-like interaction rather than recognition alone.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02449"},"ranking":{"90d":{"score":42,"rank":258,"coverage":0.7,"confidence":"Medium"}},"description":"HLL is a benchmark that evaluates multimodal agents on interactive CAPTCHA verification in a closed-loop GUI environment, covering diverse task types such as text transcription, image selection, sliders, jigsaw puzzles, and logic-based challenges. Scoring is based on task completion and trace-conditioned validation.","whyItMatters":"HLL addresses the gap in measuring agent capability at automation-protected workflows, providing a testbed for comparing progress in human-like interaction and process consistency.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3d31bcb2387de283f6c8469fb538a05a6a19fbf95c8790d3508dab3d7df214c1"},"motivation":"Multimodal agents are increasingly expected to operate interfaces on behalf of users, raising a central deployment question: can they truly substitute for humans in workflows that services deliberately protect against automation?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02449","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Xinhao S0101","organizationType":"community","sourceUrl":"https://github.com/XinhaoS0101/HLL","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_hero-s-journey_260791d3","familyId":"bmf_c5187f03b592","name":"HERO'S JOURNEY","oneLine":"Tests rule induction in goal-directed episodic tasks through text games, covering eight tasks across attribute and procedural induction families with controllable lexical grounding and identifiability conditions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02556","pdf":"https://arxiv.org/pdf/2606.02556","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02556"},"evidence":{"snippet":"We introduce HERO'S JOURNEY, a benchmark for rule induction in goal-directed episodic tasks, where agents must infer hidden rules from demonstrations and act on them through multi-step execution.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02556"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Tests rule induction in goal-directed episodic tasks through text games, covering eight tasks across attribute and procedural induction families with controllable lexical grounding and identifiability conditions.","whyItMatters":"Evaluates a specific cognitive capability in LLMs, revealing limitations in procedural induction and execution bottlenecks. Provides a structured testbed for studying rule learning and guiding method development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"f2300d37789974d7bc2d7c042e56275c010d65a35c8714a823251140a5f065fa"},"motivation":"We introduce HERO'S JOURNEY, a benchmark for rule induction in goal-directed episodic tasks, where agents must infer hidden rules from demonstrations and act on them through multi-step execution.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02556","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HERO'S JOURNEY Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.02556","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_docformbench_158426e5","familyId":"bmf_7b27e2f9f67b","name":"DocFormBench","oneLine":"DocFormBench evaluates content-aware document formatting for LLMs and multimodal models, using accuracy and efficiency metrics, with a workflow method DocFormFlow for target localization and modification.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.01936","pdf":"https://arxiv.org/pdf/2606.01936","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01936"},"evidence":{"snippet":"This content-aware setting remains challenging and underexplored, primarily due to the lack of dedicated evaluation datasets.To enable evaluation in realistic content-aware scenarios, we introduce DocFormBench, a benchmark that extends Text-to-Format evaluation to diverse formatting requirements, along with metrics for both accuracy and efficiency.To mitigate redundant document reading in existing methods during formatting, we propose DocFormFlow, a workflow formatting method that decouples targ","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01936"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DocFormBench evaluates content-aware document formatting for LLMs and multimodal models, using accuracy and efficiency metrics, with a workflow method DocFormFlow for target localization and modification.","whyItMatters":"Existing formatting benchmarks lack content-aware evaluation, leaving a gap in assessing target identification. This benchmark addresses that by providing diverse formatting requirements and metrics for accuracy and efficiency, enabling practical comparison of formatting models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0920d8cee2885bea75469eb16ce61d085ba8678b37b7ce072f88a79cc65ce840"},"motivation":"Recent advances in large language models (LLMs) have opened up new possibilities for automated document formatting.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01936","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cultureforest_c27631e8","familyId":"bmf_2e6f8f580779","name":"CultureForest","oneLine":"CultureForest is a benchmark for cultural norm grounded reasoning, featuring 5,378 examples across 8 domains and 53 countries/regions, with progressive evaluation from multiple-choice to open-ended generation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01879","pdf":"https://arxiv.org/pdf/2606.01879","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01879"},"evidence":{"snippet":"To bridge this gap, we introduce CultureForest, a benchmark for \\textit{Cultural Norm Grounded Reasoning}.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01879"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"CultureForest is a benchmark for cultural norm grounded reasoning, featuring 5,378 examples across 8 domains and 53 countries/regions, with progressive evaluation from multiple-choice to open-ended generation.","whyItMatters":"The benchmark addresses the evaluation gap between cultural knowledge and its application, offering a verifiable and attributable assessment of reasoning grounded in cultural norms.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2902e9aeb3a587b171ece7f01312000fc53ec4b804513780ce9c1c48b8e84809"},"motivation":"Existing research largely reduces cultural intelligence in LLMs to a knowledge-level problem, overlooking whether models can effectively utilize their acquired knowledge in realistic scenarios.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01879","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_crab-bench_a7bbb55e","familyId":"bmf_5a353a7a7b36","name":"CRAB-Bench","oneLine":"CRAB-Bench evaluates LLM agents on tasks generated via a constraint graph over multiple interdependent entities, using the RUSE user simulator with imperfect behavior. Scoring is pass@1 against valid solutions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01815","pdf":"https://arxiv.org/pdf/2606.01815","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01815"},"evidence":{"snippet":"We introduce CRAB-Bench (Constraint-based Realistic Agent Benchmark) and RUSE (Realistic User Simulation Engine) to address this gap.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01815"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"CRAB-Bench evaluates LLM agents on tasks generated via a constraint graph over multiple interdependent entities, using the RUSE user simulator with imperfect behavior. Scoring is pass@1 against valid solutions.","whyItMatters":"Establishes an evaluation setting for agent performance under complex task dependencies and simulated realistic user behavior, measuring task-solving ability and conversational quality in service scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b466f9c6d77fa5219d3edbe8f546d6698546f0636cccd8a51e3b645b4fb9c869"},"motivation":"Evaluating LLM agents in realistic service scenarios requires complex task dependencies, imperfect user behavior, and an evaluation that accommodates multiple valid solutions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01815","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_continuousbench_3f96ea58","familyId":"bmf_d6ffcf4f16b1","name":"ContinuousBench","oneLine":"ContinuousBench evaluates differentially private synthetic text by measuring capability gain on QA sets derived from a new quarterly corpus (Geminon procedural data or News articles), with standardized training and evaluation harness.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01849","pdf":"https://arxiv.org/pdf/2606.01849","project":"https://peihanliu.com/posts/continuousbench.html","code":"https://github.com/plau666/ContinuousBenchEval","data":null,"hfPaper":"https://huggingface.co/papers/2606.01849"},"evidence":{"snippet":"Thus, we introduce ContinuousBench, a continuously and automatically-regenerated benchmark that measures capability gain from DP synthetic text.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01849"},"ranking":{"90d":{"score":49,"rank":208,"coverage":0.7,"confidence":"Medium"}},"description":"ContinuousBench evaluates differentially private synthetic text by measuring capability gain on QA sets derived from a new quarterly corpus (Geminon procedural data or News articles), with standardized training and evaluation harness.","whyItMatters":"Addresses the gap where existing benchmarks are nearly solvable without corpus access, providing a continuously regenerated test to determine whether DP synthesis transmits genuinely new knowledge and capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"113e95b484f3adde67025ea460c049ad054b99d2741702d34dc1c2b12308965c"},"motivation":"Differentially private (DP) text synthesis promises to unlock sensitive corpora for model training, but it remains unclear whether DP synthetic data transmits genuinely new knowledge and capabilities present only in those corpora.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01849","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ContinuousBench Team","organizationType":"academic-lab","sourceUrl":"https://github.com/plau666/ContinuousBenchEval","role":"benchmark-publisher"},{"name":"Peihan Liu","organizationType":"academic-lab","sourceUrl":"https://peihanliu.com/posts/continuousbench.html","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_codegenbench_b6866f0c","familyId":"bmf_25ab5218d3f7","name":"CodegenBench","oneLine":"CodegenBench evaluates LLM-generated parallel code across x86_64, Sunway, and Kunpeng architectures using BLAS routines and specialized kernels. Scoring measures efficiency on each platform.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.04023","pdf":"https://arxiv.org/pdf/2606.04023","project":"https://anonymous.4open.science/r/CodegenBench-EDE1/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.04023"},"evidence":{"snippet":"To bridge this gap, we introduce CodegenBench, a comprehensive benchmark suite designed to evaluate the generation of efficient parallel code across three distinct hardware platforms: x86_64, Sunway, and Kunpeng.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.04023"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CodegenBench evaluates LLM-generated parallel code across x86_64, Sunway, and Kunpeng architectures using BLAS routines and specialized kernels. Scoring measures efficiency on each platform.","whyItMatters":"Addresses the gap in evaluating code generation for CPU-oriented HPC platforms, revealing cross-platform generalization limitations that matter for deploying LLMs in supercomputing contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"13e8dba48bbcffc07c2b92d947ad8cb04d4c16124dcef3c28a377910ee231e54"},"motivation":"While large language models (LLMs) have been extensively evaluated on code generation tasks for general-purpose programming and GPU-accelerated environments (e.g., PyTorch, CUDA), their capabilities in CPU-oriented high-performance computing (HPC) across diverse architectures remain underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.04023","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_clinenv_3008fd36","familyId":"bmf_e93c43eae3f3","name":"ClinEnv","oneLine":"ClinEnv evaluates LLMs as physicians in an interactive multi-stage EHR simulation, requiring queries to four specialized agents before committing to medical decisions, scored via ontology-grounded matching.","area":"Agents & Tool Use","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02568","pdf":"https://arxiv.org/pdf/2606.02568","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02568"},"evidence":{"snippet":"We present ClinEnv, an interactive benchmark that evaluates LLMs as attending physicians over real inpatient admissions under a paradigm we term Longitudinal Inpatient Simulation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02568"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ClinEnv evaluates LLMs as physicians in an interactive multi-stage EHR simulation, requiring queries to four specialized agents before committing to medical decisions, scored via ontology-grounded matching.","whyItMatters":"Measures both decision quality and information-gathering process, exposing a gap between them that outcome-only evaluation misses, which is critical for clinical decision support.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"773ede85e4bd39a070cd8c88304b2a93444095206df712d0d4d767abb48f0123"},"motivation":"Clinical practice is not the selection of an answer from enumerated options: a physician gathers heterogeneous information incrementally and commits to sequential, irreversible decisions under uncertainty.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02568","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_citytrajbench_4320b29c","familyId":"bmf_b181f74b4134","name":"CityTrajBench","oneLine":"CityTrajBench evaluates city-scale vehicle trajectory generation methods with standardized preprocessing, model adaptation, and multi-level evaluation across statistical, VAE, GAN, diffusion, and flow-matching models.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02287","pdf":"https://arxiv.org/pdf/2606.02287","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02287"},"evidence":{"snippet":"To address this issue, we present CityTrajBench, a unified benchmark framework and protocol for city-scale vehicle trajectory generation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02287"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CityTrajBench evaluates city-scale vehicle trajectory generation methods with standardized preprocessing, model adaptation, and multi-level evaluation across statistical, VAE, GAN, diffusion, and flow-matching models.","whyItMatters":"Provides a unified protocol to compare trajectory generators under common settings, addressing fragmentation and enabling reproducible assessment of multi-objective quality trade-offs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"505221f82f6c6c3ab5495c464edfebf9537e7809068553675b16f3fe615a62cd"},"motivation":"Urban trajectory generation is a fundamental task for transportation simulation, urban planning, and mobility analytics.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02287","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_causal-plan-bench_5fd0e720","familyId":"bmf_d4b6f2500fcb","name":"Causal-Plan-Bench","oneLine":"Causal-Plan-Bench evaluates embodied planning across four causal dimensions (executability, effects, composition, robustness) using 1,200 instances from 12 tasks, with MCQ and rubric-based scoring.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Planning"],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01810","pdf":"https://arxiv.org/pdf/2606.01810","project":null,"code":"https://github.com/THUSI-Lab/Causal-Reasoner","data":null,"hfPaper":"https://huggingface.co/papers/2606.01810"},"evidence":{"snippet":"To this end, we introduce Causal-Plan-Bench, a high-fidelity diagnostic suite curated through multi-stage verification to evaluate embodied planning across four causal dimensions.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01810"},"ranking":{"90d":{"score":47,"rank":216,"coverage":0.7,"confidence":"Medium"}},"description":"Causal-Plan-Bench evaluates embodied planning across four causal dimensions (executability, effects, composition, robustness) using 1,200 instances from 12 tasks, with MCQ and rubric-based scoring.","whyItMatters":"Targets the gap where benchmarks reward linguistic prediction over physical causal reasoning, offering a diagnostic protocol to assess genuine physical agency and support training for grounded planning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b360f841ae56a26306a2e458c474a28dfaf6d4539e212bdce2b7429ced9bd3f6"},"motivation":"Current benchmarks for embodied vision-language planning often favor linguistic next-token prediction over physically grounded next-state reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01810","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"THUSI-Lab","organizationType":"academic-lab","sourceUrl":"https://github.com/THUSI-Lab/Causal-Reasoner","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_caster-bench_6f29d748","familyId":"bmf_e1ba51a6b988","name":"CASTER-Bench","oneLine":"CASTER-Bench evaluates whether user-generated content achieves positive community resonance based on multimodal attributes, using human annotations across diverse categories.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01897","pdf":"https://arxiv.org/pdf/2606.01897","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01897"},"evidence":{"snippet":"To support this task, we release CASTER-Bench, a comprehensive human-annotated benchmark covering diverse UGC categories.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01897"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"CASTER-Bench evaluates whether user-generated content achieves positive community resonance based on multimodal attributes, using human annotations across diverse categories.","whyItMatters":"Expands quality assessment from visual fidelity to social engagement, providing a task for modeling community resonance rather than just aesthetic quality.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c503eb4b3e77695cb949d4c68ef593465d667da7daf970fa564ed592c26b493b"},"motivation":"Traditional Video Quality Assessment (VQA) focuses narrowly on aesthetic fidelity, overlooking the complex social dynamics that define quality in User-Generated Content (UGC).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01897","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_behaviorbench_fe357f72","familyId":"bmf_2d286f1be0d2","name":"BehaviorBench","oneLine":"BehaviorBench evaluates personalized decision modeling from real-world behavioral traces, with belief and trade prediction tasks from prediction-market and on-chain records across 2,000 wallets.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02798","pdf":"https://arxiv.org/pdf/2606.02798","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02798"},"evidence":{"snippet":"We introduce \\textsc{BehaviorBench}, a benchmark for evaluating personalized decision modeling from real-world behavioral traces.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02798"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"BehaviorBench evaluates personalized decision modeling from real-world behavioral traces, with belief and trade prediction tasks from prediction-market and on-chain records across 2,000 wallets.","whyItMatters":"Provides a real-world alternative to simulated user benchmarks, testing whether personalization methods can use observed behavioral evidence effectively in decision-support settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6a9673298d210b2353335100da0e3da5db9c69a2e0806c42a7210fb2617580f5"},"motivation":"Many decision-support settings require systems that adapt to individual users, but evaluation data for this problem remain limited.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02798","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_avi-bench_16159c13","familyId":"bmf_b71e730eed92","name":"AVI-Bench","oneLine":"AVI-Bench evaluates omni-multimodal LLMs on audio-visual tasks across perception, understanding, and reasoning stages, with an extension probing primitive audio-visual sensation using unfamiliar low-semantic stimuli.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07643","pdf":"https://arxiv.org/pdf/2606.07643","project":"https://fudancvl.github.io/AVI-Bench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07643"},"evidence":{"snippet":"We introduce AVI-Bench, a cognitively inspired benchmark that evaluates Omni-MLLMs across three stages, perception, understanding, and reasoning, through cross-modal tasks requiring joint audio-visual interpretation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07643"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AVI-Bench evaluates omni-multimodal LLMs on audio-visual tasks across perception, understanding, and reasoning stages, with an extension probing primitive audio-visual sensation using unfamiliar low-semantic stimuli.","whyItMatters":"Provides a structured way to measure joint audio-visual capabilities in multimodal models, which is currently under-evaluated.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"01ad0a58264e4b4cff6b1e56d3583d66a3ad3c244f024bc0848fa314a3887ea3"},"motivation":"Recent advances in Omni-Multimodal Large Language Models (Omni-MLLMs) have enabled strong integration of vision, audio, and language.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07643","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_automedbench_61e5be52","familyId":"bmf_bbd322eeddb0","name":"AutoMedBench","oneLine":"AutoMedBench evaluates autonomous AI agents on end-to-end medical-AI research tasks spanning segmentation, image enhancement, VQA, report generation, and lesion detection. Tasks follow a five-stage workflow (Plan, Setup, Validate, Inference, Submit), with scoring based on both final task performance and stage-level rubric scores.","area":"Agents & Tool Use","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Agents","Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01961","pdf":"https://arxiv.org/pdf/2606.01961","project":null,"code":"https://github.com/AutoMedBench/AutoMedBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.01961"},"evidence":{"snippet":"To address this gap, we present AutoMedBench, a workflow-aware benchmark for autonomous medical-AI research across diverse medical imaging and multimodal inference tasks, organizing agent execution into a unified five-stage workflow (S1-S5): Plan, Setup, Validate, Inference, and Submit.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":28,"hfDailySubmittedAt":"2026-06-03T00:00:00.000Z","githubStars":58,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01961"},"ranking":{"90d":{"score":77,"rank":32,"coverage":0.7,"confidence":"Medium"}},"description":"AutoMedBench evaluates autonomous AI agents on end-to-end medical-AI research tasks spanning segmentation, image enhancement, VQA, report generation, and lesion detection. Tasks follow a five-stage workflow (Plan, Setup, Validate, Inference, Submit), with scoring based on both final task performance and stage-level rubric scores.","whyItMatters":"Existing medical agent benchmarks focus on final outputs, obscuring failure points. AutoMedBench provides granular stage-level scoring and error analysis, enabling targeted assessment of agent capabilities and highlighting bottlenecks like verification and submission, which can guide improvement priorities in automated medical research systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8b76663e82520ad458808d2fdc47e2e21c699407edea64cc4bc7e6c2b7db4452"},"motivation":"Autonomous agents are increasingly expected to support end-to-end medical-AI research workflows, moving beyond isolated prediction tasks or short-form clinical question answering.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01961","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_agentredbench_2025428a","familyId":"bmf_1ff4fb25b17b","name":"AgentRedBench","oneLine":"AgentRedBench evaluates LLM agents against indirect prompt injection and underspecified-authorization attacks across 24 enterprise SaaS integrations. It defines 215 attack scenarios with immutable versioning, and tracks attack success rate (ASR) for models and defenses. Open-source codebase and schemas enable replay.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CR"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-06-01","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02240","pdf":"https://arxiv.org/pdf/2606.02240","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02240"},"evidence":{"snippet":"We introduce AGENTREDBENCH, a dynamic LLM-driven redteaming benchmark of 215 subtle underspecified-authorization scenarios across 24 enterprise integrations and five attack types.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02240"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AgentRedBench evaluates LLM agents against indirect prompt injection and underspecified-authorization attacks across 24 enterprise SaaS integrations. It defines 215 attack scenarios with immutable versioning, and tracks attack success rate (ASR) for models and defenses. Open-source codebase and schemas enable replay.","whyItMatters":"Prior agent-security benchmarks cover limited integrations with static payloads, understating real-world exploitability. AgentRedBench provides a dynamic, maintainer-mediated evaluation to compare model and guard-rail effectiveness against evolving injection threats, supporting procurement and hardening decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"589fc791f1a241604065026b182fcb2f17bb478b63f04e4d447c9ec10588c3de"},"motivation":"Indirect prompt injection in tool-use agents is a concrete production threat: LLM agents read from integrations (third-party services such as Gmail, Salesforce, or Jira accessed through tool calls) whose response content the user neither writes nor controls.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02240","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_tukabench_beb4bae4","familyId":"bmf_65c1dc6edc86","name":"TukaBench","oneLine":"TUKABENCH is a benchmark for jailbreak evaluation in seven African languages, extending JailbreakBench with human translations, cultural adaptations, and code-switched prompts. It assesses LLM safety in low-resource languages using metrics like Refused, Jailbroken, and Deflection, with human validation of LLM-as-a-judge.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01322","pdf":"https://arxiv.org/pdf/2606.01322","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01322"},"evidence":{"snippet":"We introduce TUKABENCH, a jailbreak benchmark for seven African languages that extends JailbreakBench (JBB) beyond direct translation through four settings: human translation of JBB prompts, English adaptation to African contexts followed by human translation, human-curated prompts validated through interactions with GPT-5.2, and code-switched prompts combining English and African languages, isolating the effect of language, cultural grounding, and prompt evasiveness on model safety.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01322"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"TUKABENCH is a benchmark for jailbreak evaluation in seven African languages, extending JailbreakBench with human translations, cultural adaptations, and code-switched prompts. It assesses LLM safety in low-resource languages using metrics like Refused, Jailbroken, and Deflection, with human validation of LLM-as-a-judge.","whyItMatters":"The benchmark addresses a gap in safety evaluation for low-resource African languages, showing that models are more vulnerable to jailbreak prompts in these languages. It could inform safer deployment of LLMs in multilingual contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"10d2859030b484f55598584f77c60bfaeaac1940867c15577f58e92d3b7d502c"},"motivation":"Safety evaluation of Large Language Models (LLMs) remains heavily English-centric, leaving Low-Resource Languages (LRLs), particularly African ones, critically underexplored.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01322","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_traveleval_64e034c5","familyId":"bmf_281eeec52056","name":"TravelEval","oneLine":"TravelEval evaluates LLM-powered travel planning agents in a realistic sandbox with accommodation pricing and intercity transport data, using six dimensions: accuracy, compliance, temporality, spatiality, economy, and utility. It simulates complete plans with API-integrated geographic information and queuing time.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01046","pdf":"https://arxiv.org/pdf/2606.01046","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01046"},"evidence":{"snippet":"To address this gap, we introduce TravelEval, a realistic and comprehensive benchmark.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01046"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TravelEval evaluates LLM-powered travel planning agents in a realistic sandbox with accommodation pricing and intercity transport data, using six dimensions: accuracy, compliance, temporality, spatiality, economy, and utility. It simulates complete plans with API-integrated geographic information and queuing time.","whyItMatters":"Existing travel planning benchmarks overemphasize constraint compliance, lack real-world data coverage, and miss global plan quality. TravelEval offers a multi-dimensional framework to compare agent planning capabilities, aiding model selection for complex travel tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7a91bfc7574eee24a3fb9131e510d2ff6b23b941211c46e5635638b2fdcb83ce"},"motivation":"The development of Large Language Models (LLMs) has significantly improved travel planning applications, yet evaluating such models is limited by existing benchmarks' limitations: 1) overemphasis on constraint compliance, neglecting multi-dimensional qualities like spatio-temporal cost; 2) datasets lacking real-world authenticity and coverage in key areas (e.g., lodging, transport); and 3) isolated daily plan assess…","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"KDD 2026","evidence":"31pages, 8 figures, accepted by KDD 2026","evidenceUrl":"https://arxiv.org/abs/2606.01046","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"KDD 2026","reviewStatus":"accepted","decisionRaw":"31pages, 8 figures, accepted by KDD 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.01046","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"31pages, 8 figures, accepted by KDD 2026","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_timesage-mt_04a07ed7","familyId":"bmf_2c4a9390c0f0","name":"TimeSage-MT","oneLine":"TimeSage-MT evaluates agentic time series reasoning in multi-turn dialogues, covering 240 tasks and 2,680 turns across 8 domains, with a protocol and leaderboard for comparing systems.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01498","pdf":"https://arxiv.org/pdf/2606.01498","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01498"},"evidence":{"snippet":"In this work, we introduce TimeSage-MT, a multi-turn benchmark for agentic time series reasoning with 240 tasks and 2,680 dialogue turns across 8 real-world domains, spanning basic exploration to decision-oriented analysis.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01498"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TimeSage-MT evaluates agentic time series reasoning in multi-turn dialogues, covering 240 tasks and 2,680 turns across 8 domains, with a protocol and leaderboard for comparing systems.","whyItMatters":"It fills a gap in benchmarking agentic time series analysis, which differs from single-step tasks, offering a way to assess multi-turn memory, uncertainty handling, and decision-making, with practical value for developing and comparing LLM agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8c6f9a4db9e9b65d32eb16ae6ba3c7b366f31a910e1e8ceda803b20674bc94c5"},"motivation":"Time series data inform critical decisions across many real-world domains.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01498","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"TimeSage-MT Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.01498","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_svhalluc_d54e2d9e","familyId":"bmf_d73ceabecf48","name":"SVHalluc","oneLine":"SVHalluc evaluates speech-vision hallucination in audio-visual large language models, focusing on semantic and temporal alignment between speech content and visual signals.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["eess.AS"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02642","pdf":"https://arxiv.org/pdf/2606.02642","project":"https://chenshuang-zhang.github.io/projects/svhalluc/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02642"},"evidence":{"snippet":"To systematically study this, we introduce SVHalluc, the first comprehensive benchmark for evaluating speech-vision hallucination in audio-visual LLMs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02642"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"SVHalluc evaluates speech-vision hallucination in audio-visual large language models, focusing on semantic and temporal alignment between speech content and visual signals.","whyItMatters":"Addresses a critical gap in evaluating audio-visual LLMs, as prior benchmarks ignored speech-induced hallucinations. Provides a systematic method to assess model grounding, aiding in model development and selection for multimodal applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f1bed85e3bfe08a642a1a3922c8487c37af6d1c0eacd88b9a6a4d7002c669bed"},"motivation":"Despite the success of audio-visual large-language models (LLMs), they can produce plausible but ungrounded outputs, termed hallucination.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"CVPR 2026","evidence":"Accepted at CVPR 2026","evidenceUrl":"https://arxiv.org/abs/2606.02642","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"CVPR 2026","reviewStatus":"accepted","decisionRaw":"Accepted at CVPR 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.02642","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at CVPR 2026","level":"author-claim"}]}],"publishers":[{"name":"Chenshuang Zhang et al.","organizationType":"academic-lab","sourceUrl":"https://chenshuang-zhang.github.io/projects/svhalluc/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_safegen-bench_15e57f70","familyId":"bmf_dca094d6bfad","name":"SafeGen-Bench","oneLine":"Evaluates safety of conditional text-to-video generation using selected start frames and text prompts across 10 malicious categories, measuring unsafety scores and guardrail effectiveness.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.01481","pdf":"https://arxiv.org/pdf/2606.01481","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01481"},"evidence":{"snippet":"To bridge this gap, we introduce SafeGen-Bench, a benchmark specifically designed to evaluate the safety of conditional T2V models.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01481"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates safety of conditional text-to-video generation using selected start frames and text prompts across 10 malicious categories, measuring unsafety scores and guardrail effectiveness.","whyItMatters":"Addresses the gap of safety evaluation when both text and image inputs are benign but output is harmful. Provides a benchmark to improve model safeguards in dynamic video generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"49841dd9018d5cac29fb4840d4e1bb32d38dfc9217d923b3c86c393b51e76e09"},"motivation":"With the rapid advancements in text-to-image diffusion models, generative video models (T2V models) like Sora can now produce short synthetic videos from a text prompt or an initial image.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01481","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SafeGen-Bench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.01481","role":"benchmark-publisher"}],"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_saber_ac2e2d88","familyId":"bmf_e4c3c153699b","name":"SABER","oneLine":"SABER evaluates operational safety of LLM coding agents in stateful project workspaces. Agents perform realistic tasks, and safety is scored from the final environment state after action sequences. Violations are categorized by cause, enabling model-specific safety profile analysis.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents","Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01317","pdf":"https://arxiv.org/pdf/2606.01317","project":null,"code":"https://github.com/sssr-lab/saber","data":null,"hfPaper":"https://huggingface.co/papers/2606.01317"},"evidence":{"snippet":"We present SABER, a benchmark for environment-aware operational safety that places models in realistic agent-style projects and evaluates safety from the final environment state after a sequence of actions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":"2026-06-05T00:00:00.000Z","githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01317"},"ranking":{"90d":{"score":50,"rank":194,"coverage":0.7,"confidence":"Medium"}},"description":"SABER evaluates operational safety of LLM coding agents in stateful project workspaces. Agents perform realistic tasks, and safety is scored from the final environment state after action sequences. Violations are categorized by cause, enabling model-specific safety profile analysis.","whyItMatters":"Existing safety benchmarks only check prompt refusal, missing the impact of action sequences on workspaces. SABER fills this gap by measuring environment-aware operational safety, offering a practical way to compare models on their ability to avoid harmful state changes in realistic coding tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1e421050f57d308d4abc8168731c709ad0782aec2be5df5be06bd3189bfd7a0e"},"motivation":"Large language models are increasingly deployed as coding agents, shifting safety from individual responses to action sequences.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01317","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SSSR Lab","organizationType":"academic-lab","sourceUrl":"https://github.com/sssr-lab/saber","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_refmem-bench_c6cd0aef","familyId":"bmf_06cc5544fe08","name":"RefMem-Bench","oneLine":"RefMem-Bench is a benchmark for reflective memory in long-horizon dialogue, containing 26K QA instances across eight reflective-memory dimensions and three task formats.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01223","pdf":"https://arxiv.org/pdf/2606.01223","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01223"},"evidence":{"snippet":"To address this gap, we introduce RefMem-Bench, a benchmark for reflective memory in long-horizon dialogue.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01223"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RefMem-Bench is a benchmark for reflective memory in long-horizon dialogue, containing 26K QA instances across eight reflective-memory dimensions and three task formats.","whyItMatters":"It evaluates the ability of models to synthesize fragmented multimodal cues into high-level interpretations, going beyond explicit recall.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"46107058f8122d18b9afcdd06180ee89c64834ec4a1427d54a4184e6709cd9e0"},"motivation":"Despite substantial progress in long-context modeling, existing benchmarks remain confined to factual memory for explicit recall, failing to measure the reflective memory required to synthesize fragmented, multimodal cues into high-level interpretations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01223","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_polyspeech-100_ce0b7268","familyId":"bmf_0150fcfdb2d3","name":"PolySpeech-100","oneLine":"PolySpeech-100 evaluates speech understanding in speech-large language models across 110 linguistic variants, including 19 Chinese dialects and over 80 low-resource languages, using tasks that assess semantic reasoning beyond transcription.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01016","pdf":"https://arxiv.org/pdf/2606.01016","project":null,"code":"https://github.com/YoungSeng/PolySpeech-100","data":null,"hfPaper":"https://huggingface.co/papers/2606.01016"},"evidence":{"snippet":"To bridge this gap, we introduce PolySpeech-100, a massive-scale benchmark designed to assess `native-level' speech comprehension across 110 linguistic variants.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01016"},"ranking":{"90d":{"score":49,"rank":202,"coverage":0.7,"confidence":"Medium"}},"description":"PolySpeech-100 evaluates speech understanding in speech-large language models across 110 linguistic variants, including 19 Chinese dialects and over 80 low-resource languages, using tasks that assess semantic reasoning beyond transcription.","whyItMatters":"Existing speech benchmarks are biased toward high-resource languages and focus on low-level recognition, limiting assessment of reasoning abilities and dialect robustness. PolySpeech-100 provides a broader coverage and a scoring protocol for comparing model performance on diverse speech understanding tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e48faa79f37ee5f2f409c8154817d35081d1eb784b978a44cea4621c957ad93e"},"motivation":"While End-to-End (E2E) Speech-Large Language Models (Speech-LLMs) are rapidly evolving, their evaluation methodologies remain limited to the era of simple transcription.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01016","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"PolySpeech-100 Benchmark Team","organizationType":"academic-lab","sourceUrl":"https://github.com/YoungSeng/PolySpeech-100","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_medgym_d9be6475","familyId":"bmf_0a3477dc88fd","name":"MedGym","oneLine":"Provides a continuous-time reinforcement learning environment for dynamic medical treatment recommendation, built from clinical data with physics-informed neural networks. Supports offline and online RL with configurable parameters.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01028","pdf":"https://arxiv.org/pdf/2606.01028","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01028"},"evidence":{"snippet":"To address this gap, we introduce MedGym, a benchmark environment for dynamic treatment recommendation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01028"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Provides a continuous-time reinforcement learning environment for dynamic medical treatment recommendation, built from clinical data with physics-informed neural networks. Supports offline and online RL with configurable parameters.","whyItMatters":"Offers a realistic testbed for RL methods in continuous-time medical settings, enabling evaluation of personalization, safety, and online deployment gaps. Standardizes comparisons between discrete and continuous time approaches.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"95a9aebde44e8244c4f90094756fbefb76d4dfde9aa581d721d7573e9a406ab4"},"motivation":"Medical treatment recommendation poses several challenges to reinforcement learning (RL): patient physiology evolves in continuous time, measurements and interventions are performed at irregular intervals, and treatment effects vary substantially across individuals.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01028","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MedGym Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.01028","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_hakushobench_550d3cd1","familyId":"bmf_40c99bc08d37","name":"HakushoBench","oneLine":"HakushoBench is a Japanese chart and table VQA benchmark built from 33 governmental white papers, containing 2,053 images across over 10 image types with manually annotated QA pairs. It evaluates vision-language models on deep holistic understanding of charts and tables.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.01132","pdf":"https://arxiv.org/pdf/2606.01132","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.01132"},"evidence":{"snippet":"As a first instantiation, we introduce HakushoBench, a challenging Japanese chart and table VQA benchmark built from 33 governmental white papers.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":5,"hfDailySubmittedAt":"2026-06-02T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01132"},"ranking":{"90d":{"score":52,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"HakushoBench is a Japanese chart and table VQA benchmark built from 33 governmental white papers, containing 2,053 images across over 10 image types with manually annotated QA pairs. It evaluates vision-language models on deep holistic understanding of charts and tables.","whyItMatters":"HakushoBench addresses the scarcity of non-English benchmarks for chart and table understanding, providing a challenging evaluation for VLMs in Japanese document domains. It reveals a notable performance gap between open-weight and proprietary models, highlighting areas for improvement in multilingual document AI.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"378f519992e7e3b1ad147dd8fbcef2ebcecc8c213ba809501addb3b6ae57375c"},"motivation":"Understanding chart and table images is essential for applying vision-language models (VLMs) to real-world document understanding.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01132","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HakushoBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2606.01132","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_chartarena_4eca7a75","familyId":"bmf_9262a71bdf07","name":"ChartArena","oneLine":"ChartArena is a bilingual benchmark for chart parsing covering eight chart families (bar, line, pie, radar, box plot, combination, flowchart, mind map) across three visual scenarios (digital, printed, hand-drawn). It uses a format-agnostic evaluation protocol with structure-aware metrics.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01348","pdf":"https://arxiv.org/pdf/2606.01348","project":null,"code":"https://github.com/pspdada/ChartArena","data":null,"hfPaper":"https://huggingface.co/papers/2606.01348"},"evidence":{"snippet":"To address these issues, we introduce ChartArena, a comprehensive bilingual benchmark covering eight chart families spanning both numeric charts and diagrammatic structures, each evaluated across three visual scenarios: digital renderings, printed photos, and hand-drawn photos.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":"2026-06-02T00:00:00.000Z","githubStars":8,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01348"},"ranking":{"90d":{"score":57,"rank":154,"coverage":0.7,"confidence":"Medium"}},"description":"ChartArena is a bilingual benchmark for chart parsing covering eight chart families (bar, line, pie, radar, box plot, combination, flowchart, mind map) across three visual scenarios (digital, printed, hand-drawn). It uses a format-agnostic evaluation protocol with structure-aware metrics.","whyItMatters":"Existing chart benchmarks cover limited chart types and ignore diagrammatic structures and real-world images. ChartArena provides a unified evaluation across diverse charts and scenarios, enabling assessment of model generalization and identifying capability gaps.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ee977f4e45f8392a61f636322d496194ccad63a5cc654c053fe77f70975d5831"},"motivation":"Charts are a primary medium for conveying quantitative and relational information, yet systematically evaluating chart parsing models remains difficult.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01348","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ChartArena Team","organizationType":"community","sourceUrl":"https://github.com/pspdada/ChartArena","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_3dcodebench_14ee1d5d","familyId":"bmf_151d2c52aca3","name":"3DCodeBench","oneLine":"3DCodeBench evaluates vision-language model agents on procedural 3D modeling by converting text and image references into Blender Python code. It includes 212 object categories with ground-truth scripts, and scores outputs on executability, image similarity (SigLIP-2/DINOv3), 3D shape distance (Chamfer/Uni3D), and LLM-as-judge, plus a human-preference ranking platform.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-31","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.01057","pdf":"https://arxiv.org/pdf/2606.01057","project":"https://www.3dcodebench.com/","code":"https://github.com/gaoypeng/3dcodebench","data":null,"hfPaper":"https://huggingface.co/papers/2606.01057"},"evidence":{"snippet":"In this paper, we propose 3DCodeBench, a systematic benchmark for evaluating vision-language model (VLM) agents for procedural 3D generation in 3D modeling software.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":"2026-06-02T00:00:00.000Z","githubStars":80,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.01057"},"ranking":{"90d":{"score":76,"rank":38,"coverage":0.7,"confidence":"Medium"}},"description":"3DCodeBench evaluates vision-language model agents on procedural 3D modeling by converting text and image references into Blender Python code. It includes 212 object categories with ground-truth scripts, and scores outputs on executability, image similarity (SigLIP-2/DINOv3), 3D shape distance (Chamfer/Uni3D), and LLM-as-judge, plus a human-preference ranking platform.","whyItMatters":"There is no standardized way to compare model abilities in procedural 3D code generation. This benchmark provides a fixed protocol and dataset, enabling reproducible evaluation and comparison across models and coding-agent settings, useful for selecting models or guiding development of procedural modeling capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e46eeef8ddb34438972594cea22941243dba369ff169288b03aa926d38021f6a"},"motivation":"Procedural 3D modeling through code is emerging as a versatile paradigm, offering deterministic, engine-ready, and precisely editable assets that neural 3D generators inherently lack.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.01057","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Google","organizationType":"company-research-lab","sourceUrl":"https://github.com/gaoypeng/3dcodebench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_skillvetbench_809c89ae","familyId":"bmf_13b32737c884","name":"SkillVetBench","oneLine":"SkillVetBench is a two-stage security vetting benchmark for open agentic skill ecosystems, evaluating detection of malicious skills via semantic analysis and runtime verification in a sandbox. The benchmark is built from confirmed malicious skills in the OpenClaw ecosystem.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00925","pdf":"https://arxiv.org/pdf/2606.00925","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.00925"},"evidence":{"snippet":"We present SkillVetBench, a two-stage security vetting benchmark for open agentic skill ecosystems.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00925"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SkillVetBench is a two-stage security vetting benchmark for open agentic skill ecosystems, evaluating detection of malicious skills via semantic analysis and runtime verification in a sandbox. The benchmark is built from confirmed malicious skills in the OpenClaw ecosystem.","whyItMatters":"Open agent platforms face supply-chain risks from malicious skills, but existing defenses lack a standardized evaluation. This benchmark addresses the gap in measuring both detection and runtime verification, offering practical value for improving security in agent ecosystems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6cd31b8641e63a6bd86c0c4d989c9c886dbfffc6108291f43641b0a1800463bb"},"motivation":"Open agent platforms allow community contributors to publish reusable skills that agents can invoke at runtime.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00925","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_rwgbench_ed2b1ef6","familyId":"bmf_9188a660e3ea","name":"RWGBench","oneLine":"Evaluates related work generation as citation-centric scholarly positioning. Uses 100 peer-reviewed papers and a 1.09M-document retrieval corpus, with metrics for citation selection, contextual appropriateness, organization, and discourse structure.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.DL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.24894","pdf":"https://arxiv.org/pdf/2606.24894","project":null,"code":"https://github.com/BFTree/RWGBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.24894"},"evidence":{"snippet":"However, related work writing is fundamentally a citation-level scholarly positioning task: it requires selecting, organizing, and framing prior work to clarify how a target paper relates to, differs from, and contributes beyond existing research.As a result, models may generate coherent and semantically-relevant text while exhibiting academically critical failures, such as inappropriate citation selection or misplaced references, that conventional metrics do not capture.To this end, we introduc","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.24894"},"ranking":{"90d":{"score":35,"rank":307,"coverage":0.7,"confidence":"Medium"}},"description":"Evaluates related work generation as citation-centric scholarly positioning. Uses 100 peer-reviewed papers and a 1.09M-document retrieval corpus, with metrics for citation selection, contextual appropriateness, organization, and discourse structure.","whyItMatters":"Fills the gap in RWG evaluation that relies on surface text similarity, offering a citation-centric testbed that aligns with expert judgment and reveals systematic limitations in current systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a1ecf362651ed4c768e7728026c03a125a995cfb2be63e634041548dbc9f88dc"},"motivation":"Large language models have shown strong fluency in scientific writing, yet the evaluation of related work generation (RWG) remains limited.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.24894","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_protstructqa_a89c5890","familyId":"bmf_b5788fc59ce8","name":"ProtStructQA","oneLine":"ProtStructQA is an executable benchmark for protein structural question answering, with questions generated from DSL programs and answers obtained by executing on AlphaFold-predicted structures. Released 382.2K questions covering confidence, distances, PAE, solvent exposure, secondary structure, topology, and contacts.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00451","pdf":"https://arxiv.org/pdf/2606.00451","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.00451"},"evidence":{"snippet":"We introduce ProtStructQA, an executable benchmark for protein structural question answering in which each natural-language question is generated from a hidden typed domain-specific language (DSL) program and the answer is obtained by executing that program on an AlphaFold-predicted structure.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00451"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ProtStructQA is an executable benchmark for protein structural question answering, with questions generated from DSL programs and answers obtained by executing on AlphaFold-predicted structures. Released 382.2K questions covering confidence, distances, PAE, solvent exposure, secondary structure, topology, and contacts.","whyItMatters":"Provides a diagnostic testbed for when language models can map words to executable 3D structural measurements, with a denotation threshold between model sizes.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"73bb9d9469fde572881cefb5755315379cf673c5707beeabc0118ab93696644e"},"motivation":"Protein-language systems are often evaluated by whether they generate plausible biological text, but a structural question has a sharper semantics: it denotes a measurement in a 3D coordinate system.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00451","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_personatrail_ac04b54e","familyId":"bmf_70d82a445a60","name":"PersonaTrail","oneLine":"PersonaTrail evaluates personalized web agents using realistic browsing trajectories as user history, assessing preference inference and information recall. Operates in a managed open web environment with two tasks.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2607.20482","pdf":"https://arxiv.org/pdf/2607.20482","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20482"},"evidence":{"snippet":"To bridge this gap, we introduce PersonaTrail, a benchmark for personalized web agents operating in a managed open web environment.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20482"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PersonaTrail evaluates personalized web agents using realistic browsing trajectories as user history, assessing preference inference and information recall. Operates in a managed open web environment with two tasks.","whyItMatters":"Addresses the gap in web agent benchmarks by capturing personalization from raw browsing history, moving beyond fully explicit prompts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0c774c0c9f6f08299088b528a0909b2f214b1e81776afcf361493bc8a4045b92"},"motivation":"Recent advances in large language models have enabled web agents to autonomously execute complex tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20482","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_omnieeg-bench_5de66c8c","familyId":"bmf_16b11959715a","name":"OmniEEG-Bench","oneLine":"OmniEEG-Bench is a unified benchmark for EEG foundation models, organizing evaluation into six task families and unifying 54 EEG datasets. It provides a leaderboard and code repository.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00815","pdf":"https://arxiv.org/pdf/2606.00815","project":null,"code":"https://github.com/ncclab-sustech/omni-eegbench.git","data":null,"hfPaper":"https://huggingface.co/papers/2606.00815"},"evidence":{"snippet":"Here, we introduce OmniEEG-Bench, a unified benchmark and downstream task roadmap for EEG foundation models (FMs).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":19,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00815"},"ranking":{"90d":{"score":63,"rank":116,"coverage":0.55,"confidence":"Low"}},"description":"OmniEEG-Bench is a unified benchmark for EEG foundation models, organizing evaluation into six task families and unifying 54 EEG datasets. It provides a leaderboard and code repository.","whyItMatters":"Standardizes evaluation of EEG foundation models across diverse tasks, revealing scaling-law behavior and the importance of pretraining data diversity.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2b7ccf2ce1d3021c9c38a8d157a4da2d12134a41c3aa1797a6693a87761fbe24"},"motivation":"Electroencephalography (EEG) supports a variety of brain-computer interface (BCI) tasks ranging from brain-state monitoring to human-LLM interactions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00815","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_momento_3a4ed98e","familyId":"bmf_70f4f3d5e856","name":"Momento","oneLine":"Momento benchmarks persistent agentic task completion in multi-session service environments, requiring agents to resolve temporal dependencies and evolving user goals across sessions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00832","pdf":"https://arxiv.org/pdf/2606.00832","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.00832"},"evidence":{"snippet":"We introduce Momento, a benchmark for persistent agentic task completion in multi-session service environments, requiring agents to take consequential, tool-mediated actions while resolving temporal dependencies and evolving user goals across sessions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00832"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Momento benchmarks persistent agentic task completion in multi-session service environments, requiring agents to resolve temporal dependencies and evolving user goals across sessions.","whyItMatters":"Highlights the gap in agent evaluation by focusing on multi-session history and misestimation of user state, which is critical for realistic human-agent interaction.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6674810005bcb648bb98c33f2b9c2f56abd6c150c1a6a3de27e1dab3c76085ce"},"motivation":"Recent advances in agentic AI have enabled agents to complete complex tasks through tool use, reasoning, and multi-step planning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00832","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_mmdg-bench_d6d5ab19","familyId":"bmf_403771f7adde","name":"MMDG-Bench","oneLine":"MMDG-Bench is a benchmark for multi-modal domain generalization, providing two frameworks (D2M and M2D) and unified protocols across action recognition and face anti-spoofing tasks. Includes code repository.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00891","pdf":"https://arxiv.org/pdf/2606.00891","project":null,"code":"https://github.com/qszhan/MMDG-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2606.00891"},"evidence":{"snippet":"To address this, we introduce MMDG-Bench, a comprehensive benchmark featuring two foundational frameworks: DG then MML (D2M) and MML then DG (M2D).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00891"},"ranking":{"90d":{"score":44,"rank":247,"coverage":0.55,"confidence":"Low"}},"description":"MMDG-Bench is a benchmark for multi-modal domain generalization, providing two frameworks (D2M and M2D) and unified protocols across action recognition and face anti-spoofing tasks. Includes code repository.","whyItMatters":"Provides a principled foundation for MMDG research, with insights on framework choice and backbone effects, and offers design guidelines for multi-modal robustness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"69baafbd62366f6aa7e0e2e25a93a336c149636d2fd8d5d37633d4a5ff8ded9f"},"motivation":"Multi-modal Domain Generalization (MMDG) seeks to leverage complementary modalities to enhance model robustness on unseen domains.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00891","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mm-snowball_ce633f35","familyId":"bmf_17f078aecdcc","name":"MM-Snowball","oneLine":"MM-Snowball is a benchmark for diagnosing hallucination snowballing in multimodal multi-turn dialogue, with fine-grained analysis. It includes data and code via a project page.","area":"Multimodal","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00622","pdf":"https://arxiv.org/pdf/2606.00622","project":"https://frenkie-chiang.github.io/MM-Snowball","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.00622"},"evidence":{"snippet":"To address this, we introduce MM-Snowball, the first benchmark for fine-grained diagnosis of hallucination snowballing within dialogues.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00622"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MM-Snowball is a benchmark for diagnosing hallucination snowballing in multimodal multi-turn dialogue, with fine-grained analysis. It includes data and code via a project page.","whyItMatters":"Addresses the lack of benchmarks for error propagation in long-horizon interactions, and shows existing mitigation methods are ineffective, motivating new approaches.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"34a3475b761d9810c0eb1b66e2adde6c1559b2de348d06a491b1b222eed3cdfe"},"motivation":"Multimodal large language models (MLLMs) demonstrate remarkable visual understanding, yet their reliability in interactive settings is severely undermined by hallucination snowballing: a phenomenon where initial errors amplify across conversational turns, leading to a collapse in coherence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"International Conference on Machine Learning (ICML 2026)","evidence":"Accepted by The International Conference on Machine Learning (ICML 2026)","evidenceUrl":"https://arxiv.org/abs/2606.00622","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"International Conference on Machine Learning (ICML 2026)","reviewStatus":"accepted","decisionRaw":"Accepted by The International Conference on Machine Learning (ICML 2026)","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.00622","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by The International Conference on Machine Learning (ICML 2026)","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_mbench_6b4d0471","familyId":"bmf_5facad24d801","name":"MBench","oneLine":"MBench is a benchmark for memory capability of video world models, decomposing into entity, environment, and causal consistency with 12 sub-dimensions. Includes code, dataset, and leaderboard.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.00793","pdf":"https://arxiv.org/pdf/2606.00793","project":"https://peanutup.github.io/MBench-project/","code":"https://github.com/study-overflow/MBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.00793"},"evidence":{"snippet":"To address this gap, we present \\textbf{MBench}, a comprehensive benchmark dedicated to quantifying and evaluating the memory capability of video world models.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":11,"hfDailySubmittedAt":"2026-06-15T00:00:00.000Z","githubStars":117,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00793"},"ranking":{"90d":{"score":78,"rank":28,"coverage":0.7,"confidence":"Medium"}},"description":"MBench is a benchmark for memory capability of video world models, decomposing into entity, environment, and causal consistency with 12 sub-dimensions. Includes code, dataset, and leaderboard.","whyItMatters":"Fills the gap in evaluating long-term state retention in video world models, providing a standardized benchmark to advance the field.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4dd57fe4a8e34045c4559d47bf92d0d04c3fa4113f3cf8c0d5a3442c3930d75b"},"motivation":"Recent advancements in video-based world models have demonstrated an unprecedented ability to synthesize high-fidelity visual sequences.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00793","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_i-webgenbench_56eafa1f","familyId":"bmf_6c242fe4bdad","name":"I-WebGenBench","oneLine":"A benchmark of 19 research papers with expert-built interactive systems for evaluating agents that convert PDFs into executable web applications.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00750","pdf":"https://arxiv.org/pdf/2606.00750","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.00750"},"evidence":{"snippet":"To evaluate this task, we introduce a benchmark of 19 research papers paired with expert-built interactive systems as ground truth.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00750"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"A benchmark of 19 research papers with expert-built interactive systems for evaluating agents that convert PDFs into executable web applications.","whyItMatters":"Supports evaluation of interactive system generation but relies on limited data and lacks documented public reuse.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6ba319af5acbf6434db3bcbd5c01a60e96db0c91ec97cea9d99e7a89ec54e587"},"motivation":"Recent advances in visual language models have enabled autonomous agents for complex reasoning, tool use, and document understanding.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00750","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_foresci_2aecb7a4","familyId":"bmf_17a312d27c7d","name":"ForeSci","oneLine":"Temporally controlled benchmark with 500 tasks across AI domains for forward-looking research judgment; includes offline knowledge bases and validation protocols.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00644","pdf":"https://arxiv.org/pdf/2606.00644","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.00644"},"evidence":{"snippet":"We introduce ForeSci, a temporally controlled benchmark for evaluating whether LLM agents can make such forward-looking research judgements from historical evidence.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":"2026-06-05T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00644"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Temporally controlled benchmark with 500 tasks across AI domains for forward-looking research judgment; includes offline knowledge bases and validation protocols.","whyItMatters":"Could support evaluation of research agents in forecasting tasks, but current evidence is insufficient to establish credibility and public accessibility.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dd070da445e2eb6887ed8ed8acf0a8838f626f18df8528670d431868d91e9ddb"},"motivation":"AI research often requires decisions before future evidence exists: which bottleneck to attack, which direction to pursue, or where a project should be positioned.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00644","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cv-arena_3558dfa5","familyId":"bmf_cc2f478af1d0","name":"CV-Arena","oneLine":"CV-Arena is a benchmark of 12K high-resolution image instruction pairs across 16 task types, with a human-AI collaborative preference protocol (Active Elo) for evaluating instructional computer vision problem solving.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00931","pdf":"https://arxiv.org/pdf/2606.00931","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.00931"},"evidence":{"snippet":"We introduce CV-Arena, an open benchmark designed to evaluate this capability at professional scales.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00931"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"CV-Arena is a benchmark of 12K high-resolution image instruction pairs across 16 task types, with a human-AI collaborative preference protocol (Active Elo) for evaluating instructional computer vision problem solving.","whyItMatters":"Addresses the gap in evaluating diverse real-image editing tasks with professional constraints, providing a scalable and traceable evaluation resource for model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5c20d7f13a506c6bcd4b096345171b0301000890ae3bb6d3c9fa6b1f4d0bbb93"},"motivation":"Instruction-guided image editing is becoming a general interface for visual work, yet existing benchmarks still focus largely on narrow appearance edits and do not fully capture the diversity of real-image tasks in professional workflows.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00931","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_cafosat_e236d412","familyId":"bmf_01528fdfb43e","name":"CAFOSat","oneLine":"CAFOSat is a dataset of over 45,000 image patches with infrastructure-level annotations for CAFO mapping across 20 states, benchmarking models for infrastructure-aware agricultural monitoring.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-05-30","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2606.00548","pdf":"https://arxiv.org/pdf/2606.00548","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.00548"},"evidence":{"snippet":"We benchmark a diverse set of convolutional, transformer-based, and vision-language models, demonstrating the value of refined annotations and curated negative samples for CAFO classification and generalization.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00548"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CAFOSat is a dataset of over 45,000 image patches with infrastructure-level annotations for CAFO mapping across 20 states, benchmarking models for infrastructure-aware agricultural monitoring.","whyItMatters":"Provides a large-scale, infrastructure-aware benchmark for advancing CAFO mapping from remote sensing, addressing the lack of strongly annotated datasets in this domain.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"50c6afb0efd9947517d58fa6465084e6d3546ec1ae8d1fcf9532b1f60ff2aadc"},"motivation":"Concentrated Animal Feeding Operations (CAFOs) play an important role in agricultural production but are also associated with environmental, public health, and disease surveillance concerns.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"CVPR Workshop-2026","evidence":"Accepted at CVPR Workshop-2026. First two authors has equal contribution","evidenceUrl":"https://arxiv.org/abs/2606.00548","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"CVPR Workshop-2026","reviewStatus":"accepted","decisionRaw":"Accepted at CVPR Workshop-2026. First two authors has equal contribution","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.00548","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at CVPR Workshop-2026. First two authors has equal contribution","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_webigbench_7eee6092","familyId":"bmf_c2c33131ad91","name":"WebIGBench","oneLine":"WebIGBench evaluates code generation for interactive webpages with 103 complex examples and 871 distinct actions, proposing an automated evaluation pipeline for interactive consistency.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation"],"topics":["Multimodal","Code"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2606.00154","pdf":"https://arxiv.org/pdf/2606.00154","project":null,"code":"https://github.com/anoa12159-hue/WebIGBench_eval","data":null,"hfPaper":"https://huggingface.co/papers/2606.00154"},"evidence":{"snippet":"To address these limitations, we introduce WebIGBench, the first benchmark designed to evaluate code generation for interactive webpages with complex interactions.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00154"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"WebIGBench evaluates code generation for interactive webpages with 103 complex examples and 871 distinct actions, proposing an automated evaluation pipeline for interactive consistency.","whyItMatters":"Fills the gap in benchmarking interactive webpage code generation, offering a public dataset and evaluation method for model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"02b84a3e54e2205d58f6e2672fdbbf08df43362a1a96f92846c3f9607760912d"},"motivation":"Recent advancements in multimodal large language models (MLLMs) have achieved remarkable progress in multimodal reasoning and code generation, catalyzing a new paradigm for front-end development.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00154","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_visualleakbench_6618ffcf","familyId":"bmf_233962516067","name":"VisualLeakBench","oneLine":"VisualLeakBench is a 500-image benchmark for evaluating action-boundary propagation failures in vision-language agents, with stratified subsets and oracle diagnostics.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents","Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07595","pdf":"https://arxiv.org/pdf/2606.07595","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.07595"},"evidence":{"snippet":"We present VisualLeakBench, a diversified 500-image benchmark spanning UI, chat, document, form, and dashboard scenes, and evaluate a stratified 100-image agent subset with four production VLM systems under two workflows: note capture and external handoff.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07595"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"VisualLeakBench is a 500-image benchmark for evaluating action-boundary propagation failures in vision-language agents, with stratified subsets and oracle diagnostics.","whyItMatters":"Targets a specific safety failure mode in VLAs, but lacks clear public reuse path and scoring contract details.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5e01a3ea88a8b3cd98e1cfcb723ffdbdd4119cb6d556b049c93f9bcd5aaaa601"},"motivation":"Vision-language agents increasingly consume screenshots, documents, and user interfaces before writing to memory, sending messages, or invoking external tools.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07595","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_tsm-bench_b1598d50","familyId":"bmf_dd5ec332da38","name":"TSM-Bench","oneLine":"TSM-Bench is a multilingual, multi-generator, multi-task benchmark for evaluating machine-generated text detectors on real-world Wikipedia editing tasks.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.31113","pdf":"https://arxiv.org/pdf/2605.31113","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.31113"},"evidence":{"snippet":"We introduce \\textsc{TSM-Bench}, a multilingual, multi-generator, and \\textit{multi-task} benchmark for evaluating MGT detectors on common, real-world Wikipedia editing tasks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.31113"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TSM-Bench is a multilingual, multi-generator, multi-task benchmark for evaluating machine-generated text detectors on real-world Wikipedia editing tasks.","whyItMatters":"Reveals performance drops in detectors on task-specific MGT, providing a foundation for developing robust detectors for UGC platforms.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c53c900004dbc2aaa4c8a856779bce8f2e452dddc6c07d96e7e33b2f83c6613b"},"motivation":"Automatically detecting machine-generated text (MGT) is critical to maintaining the knowledge integrity of user-generated content (UGC) platforms such as Wikipedia.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.31113","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_touchsafebench_f153a852","familyId":"bmf_f09b093c7c32","name":"TouchSafeBench","oneLine":"TouchSafeBench is a physics-grounded benchmark with 2,940 simulated episodes for evaluating collision grounding in vision-language models for safe human-robot collaboration.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics","Multimodal"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.31196","pdf":"https://arxiv.org/pdf/2605.31196","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.31196"},"evidence":{"snippet":"We introduce TouchSafeBench, a physics-grounded benchmark for evaluating collision grounding in vision-language models (VLMs).","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.31196"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"TouchSafeBench is a physics-grounded benchmark with 2,940 simulated episodes for evaluating collision grounding in vision-language models for safe human-robot collaboration.","whyItMatters":"Targets a critical safety capability in embodied AI, but current evidence lacks public availability and scoring details.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ef272e2854a2d8ec836707474afd698a7b5243512e65980a28624fcd5db8f7e9"},"motivation":"Safe human--robot collaboration requires more than visual description: a monitor must determine whether the robot body is safely separated, already colliding with the scene or a person, or about to collide.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.31196","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_tada-bench_769ea7c2","familyId":"bmf_0784ab615c0b","name":"TadA-Bench","oneLine":"TadA-Bench is a fixed-data replay benchmark derived from 31 wet-lab rounds of TadA directed evolution. Models rank ~1M protein, DNA, or RNA sequence variants appearing only in later rounds, with scores as Spearman, Recall@10%, and nDCG@10%.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["q-bio.QM"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02624","pdf":"https://arxiv.org/pdf/2606.02624","project":null,"code":"https://github.com/shiyegao/TadABench-1M","data":"https://huggingface.co/datasets/JinGao/TadABench-1M","hfPaper":"https://huggingface.co/papers/2606.02624"},"evidence":{"snippet":"We introduce TadA-Bench, a million-variant wet-lab replay benchmark from 31 TadA directed-evolution rounds for future-round discovery toward agentic protein engineering.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":526,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2606.02624"},"ranking":{"90d":{"score":54,"rank":174,"coverage":1.0,"confidence":"High","datasetDownloadRank":11,"datasetRankPopulation":51}},"description":"TadA-Bench is a fixed-data replay benchmark derived from 31 wet-lab rounds of TadA directed evolution. Models rank ~1M protein, DNA, or RNA sequence variants appearing only in later rounds, with scores as Spearman, Recall@10%, and nDCG@10%.","whyItMatters":"Standard random-split evaluation overestimates performance for iterative candidate prioritization. TadA-Bench provides a chronological replay protocol to measure whether models can transfer from earlier experimental rounds to future ones, a core requirement for agentic protein engineering.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a9967bd21cdc02003589cb6cb7db5ed19dcd7a95ea2a111f18834ae8f98ec307"},"motivation":"AI for scientific discovery is entering an agentic era, where protein-engineering systems are expected to prioritize future wet-lab experiments rather than merely fit static measurements.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"43rd International Conference on Machine Learning (ICML 2026)","evidence":"Accepted at the 43rd International Conference on Machine Learning (ICML 2026). Data: https://huggingface.co/datasets/JinGao/TadABench-1M . Code: https://github.com/shiyegao/TadABench-1M","evidenceUrl":"https://arxiv.org/abs/2606.02624","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"43rd International Conference on Machine Learning (ICML 2026)","reviewStatus":"accepted","decisionRaw":"Accepted at the 43rd International Conference on Machine Learning (ICML 2026). Data: https://huggingface.co/datasets/JinGao/TadABench-1M . Code: https://github.com/shiyegao/TadABench-1M","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2606.02624","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at the 43rd International Conference on Machine Learning (ICML 2026). Data: https://huggingface.co/datasets/JinGao/TadABench-1M . Code: https://github.com/shiyegao/TadABench-1M","level":"author-claim"}]}],"publishers":[{"name":"Shanghai Jiao Tong University","organizationType":"academic-lab","sourceUrl":"https://github.com/shiyegao/TadABench-1M","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_svi-bench_918464db","familyId":"bmf_17a1b917580e","name":"SVI-Bench","oneLine":"SVI-Bench evaluates vision-language models on strategic video intelligence using sports as a microworld. It includes 9 tasks across 4 pillars: perception, reasoning, simulation, and agency, using basketball, soccer, and hockey videos with annotated actions and reports.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.31529","pdf":"https://arxiv.org/pdf/2605.31529","project":null,"code":"https://github.com/Texaser/SVI-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2605.31529"},"evidence":{"snippet":"To bridge this gap, we introduce SVI-Bench, a large-scale benchmark that leverages team sports as a dynamic microworld, combining the complexity of real-world multi-agent interaction (10-22 agents making coordinated decisions under adversarial pressure) with the verifiability of explicit rules and definitive outcomes.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":7,"hfDailySubmittedAt":"2026-06-02T00:00:00.000Z","githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.31529"},"ranking":{"90d":{"score":46,"rank":225,"coverage":0.7,"confidence":"Medium"}},"description":"SVI-Bench evaluates vision-language models on strategic video intelligence using sports as a microworld. It includes 9 tasks across 4 pillars: perception, reasoning, simulation, and agency, using basketball, soccer, and hockey videos with annotated actions and reports.","whyItMatters":"Existing video benchmarks lack verifiable ground truth for causal and strategic reasoning. SVI-Bench combines real-world multi-agent complexity with verifiable rules and outcomes, enabling evaluation of higher-level cognitive capabilities in video understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"97e6cb3853ee8765a4d34720db6c57d4d38cdd6b9cd140e15205201be13bfa7b"},"motivation":"True video intelligence demands more than recognizing what is visible: it requires reasoning about why events unfold, predicting what would change under different conditions, and deciding what to do next.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.31529","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MVP Group","organizationType":"academic-lab","sourceUrl":"https://github.com/Texaser/SVI-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_stembind_bdce22aa","familyId":"bmf_140387d0197e","name":"StemBind","oneLine":"StemBind is a diagnostic benchmark with shared-stem questions to attribute failures in abstract visual reasoning. It includes perception, rule, and full tasks with stage annotations but no official code or dataset release.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00148","pdf":"https://arxiv.org/pdf/2606.00148","project":"https://hexixiang.github.io/StemBind","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.00148"},"evidence":{"snippet":"We introduce StemBind, a shared-stem diagnostic benchmark that probes the same visual stem with three aligned questions: Perception (what is in the image), Rule (what pattern governs it), and Full (which option completes it), so a final-answer error can be attributed to a specific sub-step on the same evidence.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00148"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"StemBind is a diagnostic benchmark with shared-stem questions to attribute failures in abstract visual reasoning. It includes perception, rule, and full tasks with stage annotations but no official code or dataset release.","whyItMatters":"It aims to localize reasoning failures to specific sub-steps, potentially guiding improvements in multimodal models, but lacks a public path for independent verification.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"36262cddaf9bdc858bb3fda3ebf8046138fb7d7e81302c6a1639a6ade468db6e"},"motivation":"Multimodal large language models (MLLMs) often know the rule but pick the wrong answer: on abstract visual reasoning (AVR) tasks, a model can describe what it sees and name the underlying pattern, yet still fail to choose the matching candidate.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00148","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_soco_b681a0c6","familyId":"bmf_1ea2d8f8a54b","name":"SOCO","oneLine":"SOCO is a benchmark for semantic object correspondence with keypoint annotations across 100 categories and over 1M pairs. The evaluation is implemented within OmniProbe, which includes correspondence tasks among others.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.31597","pdf":"https://arxiv.org/pdf/2605.31597","project":"https://genintel.github.io/SOCO/","code":"https://github.com/GenIntel/OmniProbe","data":null,"hfPaper":"https://huggingface.co/papers/2605.31597"},"evidence":{"snippet":"To enable a systematic SC evaluation, we introduce SOCO, a new benchmark for Semantic Object Correspondence that introduces a taxonomy of correspondence types and provides consistent, functionally meaningful keypoint annotations across 100 categories and over 1M correspondence pairs.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":"2026-06-02T00:00:00.000Z","githubStars":11,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.31597"},"ranking":{"90d":{"score":62,"rank":119,"coverage":0.7,"confidence":"Medium"}},"description":"SOCO is a benchmark for semantic object correspondence with keypoint annotations across 100 categories and over 1M pairs. The evaluation is implemented within OmniProbe, which includes correspondence tasks among others.","whyItMatters":"It provides a systematic evaluation for part-level understanding in vision models, but since it is part of a larger framework, it is not a standalone benchmark.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c1a7d3d65c4c02c3613c3ae5b1a6a2546f53601b196660c339cd8dc3ce1fdf65"},"motivation":"Measuring structured object understanding in vision foundation models remains challenging due to inconsistent evaluation protocols and limited part-level supervision.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.31597","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GenIntel","organizationType":"academic-lab","sourceUrl":"https://github.com/GenIntel/OmniProbe","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_powercodebench_c0ce0f67","familyId":"bmf_dc78daa563c7","name":"PowerCodeBench","oneLine":"PowerCodeBench is a benchmark generator for power system code generation, paired with an intervention method, but no artifacts are provided in this article.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation","Factuality"],"topics":["Code"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.31478","pdf":"https://arxiv.org/pdf/2605.31478","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.31478"},"evidence":{"snippet":"We introduce PowerCodeBench, an execution-validated benchmark generator that pairs natural-language operator queries with pandapower code and numerical ground truth; an L0-L3 documentation-driven probing procedure that measures per-model API knowledge profiles; and a boundary-aware intervention that combines query-side API demand estimation with targeted proactive documentation injection and routed reactive correction.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.31478"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PowerCodeBench is a benchmark generator for power system code generation, paired with an intervention method, but no artifacts are provided in this article.","whyItMatters":"It addresses reliability of open-weight models for on-premise deployment, but the lack of a release prevents external use.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"db350dac8e1dfdd93cb56df1e7b068a6448279e388e3944139dc9fe5222260fc"},"motivation":"Large language models (LLMs) are increasingly used to automate power-system analysis, but many utilities and energy-research labs require on-premise serving for confidentiality, regulatory, reproducibility, and cost reasons.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.31478","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_paintbench_3ad5e836","familyId":"bmf_3505fc50bfea","name":"PaintBench","oneLine":"PaintBench is a procedurally generated benchmark for precise visual editing, covering 20 tasks across geometric, structural, color, and symbolic categories. Stable scoring uses pixel-level mIoU with fixed seeds.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Reasoning","Robot manipulation"],"topics":["Robotics","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00188","pdf":"https://arxiv.org/pdf/2606.00188","project":"https://paintbench.github.io/","code":"https://github.com/PaintBench/PaintBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.00188"},"evidence":{"snippet":"To probe this challenge, we introduce PaintBench, a dynamically scalable benchmark targeting 20 fundamental precise visual editing operations across four categories: geometric transformation, structural manipulation, color change, and symbolic reasoning.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":"2026-06-04T00:00:00.000Z","githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00188"},"ranking":{"90d":{"score":51,"rank":192,"coverage":0.7,"confidence":"Medium"}},"description":"PaintBench is a procedurally generated benchmark for precise visual editing, covering 20 tasks across geometric, structural, color, and symbolic categories. Stable scoring uses pixel-level mIoU with fixed seeds.","whyItMatters":"Procedural generation allows contamination-resistant evaluation of precise editing capabilities, with deterministic scoring that avoids human or LLM bias.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3b9ce3e72b51dc69a158da08902c51bd2b3e1556ac4ed7da93c39ad4297e7cd3"},"motivation":"While current multimodal models are proficient at open-ended visual editing, executing precise single-answer edits remains an important obstacle.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00188","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"NYU","organizationType":"academic-lab","sourceUrl":"https://github.com/PaintBench/PaintBench","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_mosaicleaks_86df075c","familyId":"bmf_4525a035d410","name":"MosaicLeaks","oneLine":"MosaicLeaks is a benchmark of 1,001 multi-hop research tasks to evaluate privacy leakage in deep research agents, but no artifacts are provided in this article.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30727","pdf":"https://arxiv.org/pdf/2605.30727","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30727"},"evidence":{"snippet":"We introduce MosaicLeaks, a benchmark of 1,001 multi-hop deep research tasks that chain private enterprise documents and a public web corpus, forcing agents to make external queries that depend on local information.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30727"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MosaicLeaks is a benchmark of 1,001 multi-hop research tasks to evaluate privacy leakage in deep research agents, but no artifacts are provided in this article.","whyItMatters":"It addresses privacy risks in agentic workflows, but without a public release, external teams cannot use it.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d37259aa71e04df97907b0a570aa239b2c522b1881c6fd1a62289435fd64fca3"},"motivation":"Deep research agents increasingly combine private local documents with external tools like web retrieval, creating a privacy risk: an agent's external queries may leak sensitive information from its local context.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30727","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_grapharc_4326342c","familyId":"bmf_c87c449862ea","name":"GraphARC","oneLine":"GraphARC is a benchmark for abstract reasoning on graphs, generalizing ARC few-shot learning. No artifacts are provided in this article.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.31031","pdf":"https://arxiv.org/pdf/2605.31031","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.31031"},"evidence":{"snippet":"We introduce GraphARC, a benchmark for abstract reasoning on graph-structured data.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.31031"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GraphARC is a benchmark for abstract reasoning on graphs, generalizing ARC few-shot learning. No artifacts are provided in this article.","whyItMatters":"It aims to evaluate relational reasoning in graph models, but lacks a public path for external evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4695562d7efdbd01afb65c168459e49005fcfe4e8bfbc5bd4f76e8f69a046413"},"motivation":"Relational reasoning lies at the heart of intelligence, but existing benchmarks are typically confined to formats such as grids or text.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"KDD 2026 Datasets and Benchmarks Track","evidence":"Accepted at KDD 2026 Datasets and Benchmarks Track","evidenceUrl":"https://arxiv.org/abs/2605.31031","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"KDD 2026 Datasets and Benchmarks Track","reviewStatus":"accepted","decisionRaw":"Accepted at KDD 2026 Datasets and Benchmarks Track","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2605.31031","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at KDD 2026 Datasets and Benchmarks Track","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_fmg-bench_a89fdc7a","familyId":"bmf_9aef90db69a1","name":"FMG-Bench","oneLine":"FMG-Bench evaluates English-language Christian theological triage and pastoral guidance in LLMs. It scores whether model responses match expected behaviors for triage levels: primary doctrine, secondary doctrine, prudential, and pastoral, with safety-related escalation appropriate to each scenario. The corpus includes 120 base scenarios plus 37 perturbation variants.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CY"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2608.12324","pdf":"https://arxiv.org/pdf/2608.12324","project":null,"code":"https://github.com/FideAI/fmg-bench","data":"https://huggingface.co/datasets/FideAI/fmg-bench","hfPaper":"https://huggingface.co/papers/2608.12324"},"evidence":{"snippet":"We introduce FMG-Bench, the Faith & Moral Guidance Benchmark, a 120-scenario benchmark for evaluating large language model behavior in English-language Christian theological triage and pastoral guidance contexts.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":20,"hfDatasetLikes":1},"source":{"type":"arxiv","id":"2608.12324"},"ranking":{"90d":{"score":33,"rank":310,"coverage":0.85,"confidence":"High","datasetDownloadRank":49,"datasetRankPopulation":51}},"description":"FMG-Bench evaluates English-language Christian theological triage and pastoral guidance in LLMs. It scores whether model responses match expected behaviors for triage levels: primary doctrine, secondary doctrine, prudential, and pastoral, with safety-related escalation appropriate to each scenario. The corpus includes 120 base scenarios plus 37 perturbation variants.","whyItMatters":"Evaluates a niche but real usage area where models are asked for faith and care advice. Provides a structured scoring contract for safety-critical escalation and robustness to rephrasing, which are not covered by generic QA or safety benchmarks. Useful for developers testing models in contexts where human referral matters.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f053736d5b07605f5f849265e4b768d181e9b5d7d243468d4f37f7021760acdb"},"motivation":"People increasingly ask large language models (LLMs) for counsel on questions of faith, doctrine, and pastoral care.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.12324","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_fbhm_44ffb389","familyId":"bmf_752147a6df00","name":"FBHM","oneLine":"FBHM evaluates vision-language models on hateful meme detection across 25 rhetorical functionalities and 10 target communities, with 5,000 memes. Performance is measured by Macro-F1 score on this curated dataset.","area":"Vision & 3D","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.31349","pdf":"https://arxiv.org/pdf/2605.31349","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.31349"},"evidence":{"snippet":"To address this, we introduce FBHM, a systematically curated benchmark of Functionality Based Hateful Memes constructed along two orthogonal axes: 25 distinct rhetorical functionalities and 10 target communities (5,000 memes total).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.31349"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FBHM evaluates vision-language models on hateful meme detection across 25 rhetorical functionalities and 10 target communities, with 5,000 memes. Performance is measured by Macro-F1 score on this curated dataset.","whyItMatters":"Existing hateful meme benchmarks confound rhetorical strategies with target community features, preventing causal evaluation of model vulnerabilities. FBHM isolates these axes, revealing that models rely on dataset-specific heuristics rather than robust reasoning, and offers a controlled environment for measuring generalization.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"65fa90f444451fd25fa576d244646aea002efee6a85f595e49f3dc0616e67931"},"motivation":"Hateful meme detection remains a formidable challenge for vision-language models, as existing benchmarks are structurally observational - confounding rhetorical hate mechanisms with target community features and preventing causal evaluation of model vulnerabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.31349","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_fam-bench_1946f5cd","familyId":"bmf_07e4f08667c9","name":"FAM-Bench","oneLine":"FAM-Bench evaluates multimodal language and vision-language models on Food-as-Medicine reasoning. It includes 2500 expert-verified instances across 13 health conditions, with two tasks: dish-level suitability assessment (judging if a dish is suitable for a condition) and comparative dish analysis (ranking four dishes by suitability).","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.31410","pdf":"https://arxiv.org/pdf/2605.31410","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.31410"},"evidence":{"snippet":"We introduce FAM-Bench, a multi-modal Food-as-Medicine benchmark with 2500 nutrition-expert-verified instances across 13 diet-related health conditions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.31410"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"FAM-Bench evaluates multimodal language and vision-language models on Food-as-Medicine reasoning. It includes 2500 expert-verified instances across 13 health conditions, with two tasks: dish-level suitability assessment (judging if a dish is suitable for a condition) and comparative dish analysis (ranking four dishes by suitability).","whyItMatters":"The benchmark fills a gap in food AI evaluation by testing whether models can integrate ingredient, visual, and clinical nutrition constraints to make condition-aware food decisions. It provides a standardized testbed for comparing models on grounded health-aware reasoning, which is relevant for applications in nutrition guidance and chronic disease management.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c45bc785f974ced4e8d845a40772130131a79a51fff1fe234a5eec09872e3442"},"motivation":"Food-as-Medicine requires models to reason beyond what a dish is or what nutrition it contains: they must decide whether a concrete food choice is appropriate for a specific health condition.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.31410","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_ergeobench_37de7936","familyId":"bmf_323d61775db6","name":"ERGeoBench","oneLine":"ERGeoBench evaluates vision-driven embodied geo-localization in MLLMs with 2,207 globally distributed street-view panoramas under single-view, panorama-view, and embodied-view settings. It measures foundational perception, spatial awareness, common sense reasoning, and geo-localization reasoning.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.31251","pdf":"https://arxiv.org/pdf/2605.31251","project":"https://kaixuewen.github.io/ERGeoBench/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.31251"},"evidence":{"snippet":"We introduce ERGeoBench, a diagnostic benchmark for vision-driven embodied geo-localization.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.31251"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ERGeoBench evaluates vision-driven embodied geo-localization in MLLMs with 2,207 globally distributed street-view panoramas under single-view, panorama-view, and embodied-view settings. It measures foundational perception, spatial awareness, common sense reasoning, and geo-localization reasoning.","whyItMatters":"Embodied geo-localization is underexplored due to lack of fine-grained evaluation. ERGeoBench provides a unified diagnostic framework that reveals current MLLMs struggle with fine-grained perceptual operations and metric localization, supporting progress in integrated perception and spatial reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"19f60e4b9df0eb40d3e7709010980223210ff749e827affe8ff9644b57c89eec"},"motivation":"Multimodal large language models (MLLMs) have shown strong potential as embodied agents, yet embodied geo-localization remains underexplored due to the lack of fine-grained evaluation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.31251","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ERGeoBench Team","organizationType":"academic-lab","sourceUrl":"https://kaixuewen.github.io/ERGeoBench/","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_egostream_a12d21a7","familyId":"bmf_0c3b61ab28b8","name":"EGOSTREAM","oneLine":"Egostream evaluates streaming episodic memory in egocentric vision through 2,250 curated questions across seven cognitive dimensions, expanded into 8,528 recall-conditioned evaluations using the Answer Validity Window to test recall from instant to ultra-long-term.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.31557","pdf":"https://arxiv.org/pdf/2605.31557","project":"https://saroo25.github.io/Egostream/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.31557"},"evidence":{"snippet":"We introduce Egostream, a diagnostic benchmark for streaming episodic memory evaluation in egocentric vision.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.31557"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Egostream evaluates streaming episodic memory in egocentric vision through 2,250 curated questions across seven cognitive dimensions, expanded into 8,528 recall-conditioned evaluations using the Answer Validity Window to test recall from instant to ultra-long-term.","whyItMatters":"Existing streaming video benchmarks lack fine-grained diagnosis of memory retention over time. Egostream provides a controlled protocol that separates genuine forgetting from world-state changes, enabling precise assessment of model memory capabilities across different temporal scales.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5121561deac1f12231b29a0c829c0d07caf161ee0bf0f209877b195f3580c403"},"motivation":"Continuous episodic memory is a core capability for autonomous agents operating in dynamic, real-world environments, yet current streaming video benchmarks provide limited tools for diagnosing what models remember and for how long.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.31557","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Egostream project team","organizationType":"academic-lab","sourceUrl":"https://saroo25.github.io/Egostream/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ego-metas_beb4f202","familyId":"bmf_9a895d99bf07","name":"Ego-METAS","oneLine":"Ego-METAS evaluates online temporal action segmentation in egocentric video, where models must select sensor modalities (RGB, audio, gaze, IMU, monochrome) per timestep to maximize accuracy under hardware-representative energy budgets. It includes 100+ hours of untrimmed video from multiple datasets.","area":"Multimodal","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02246","pdf":"https://arxiv.org/pdf/2606.02246","project":"https://maria-sanvil.github.io/Ego-METAS-website/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02246"},"evidence":{"snippet":"To address this, we introduce Ego-METAS: the first Egocentric online Multimodal Energy-efficient Temporal Action Segmentation benchmark.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02246"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Ego-METAS evaluates online temporal action segmentation in egocentric video, where models must select sensor modalities (RGB, audio, gaze, IMU, monochrome) per timestep to maximize accuracy under hardware-representative energy budgets. It includes 100+ hours of untrimmed video from multiple datasets.","whyItMatters":"This benchmark addresses the gap in energy-aware perception for embodied AI by providing a standardized testbed for developing and comparing cost-aware sensor routing policies in continuous, untrimmed environments. It enables assessment of trade-offs between predictive accuracy and energy consumption, with practical implications for always-on devices.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9a24f5d4789ce7e589fc3d7a39ddabcb0363a4dc8651cd70203d5c79eea65d13"},"motivation":"To operate in the physical world, embodied agents must perceive their environment in an \"always-on\" fashion, selectively accessing the most informative sensors to balance energy constraints and task accuracy.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02246","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Ego-METAS team","organizationType":"academic-lab","sourceUrl":"https://maria-sanvil.github.io/Ego-METAS-website/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_bluefin_2fd6c32a","familyId":"bmf_98253ff66ed8","name":"BlueFin","oneLine":"Evaluates LLM agents on synthesis, manipulation, and comprehension tasks over financial spreadsheet workbooks, with 131 tasks and granular rubric criteria validated by expert annotators.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems","Finance & Economics"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics","Financial Services"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30907","pdf":"https://arxiv.org/pdf/2605.30907","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30907"},"evidence":{"snippet":"We present BlueFin, a benchmark that tasks large language model (LLM) agents with synthesis, manipulation, and comprehension tasks over spreadsheet workbooks in the professional finance domain.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30907"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Evaluates LLM agents on synthesis, manipulation, and comprehension tasks over financial spreadsheet workbooks, with 131 tasks and granular rubric criteria validated by expert annotators.","whyItMatters":"Fills the gap in evaluating LLMs for spreadsheet tasks relevant to professional finance, where current models perform below 50%, providing a measure for practical deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"90d53d7dea828e6760a3774046a5fecedc8289acee860d28ed48ca340160a8ea"},"motivation":"We present BlueFin, a benchmark that tasks large language model (LLM) agents with synthesis, manipulation, and comprehension tasks over spreadsheet workbooks in the professional finance domain.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30907","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"BlueFin Benchmark Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2605.30907","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"cross-domain"},{"id":"bm_billiardphys-bench_142a32a2","familyId":"bmf_750da7a3f066","name":"BilliardPhys-Bench","oneLine":"Evaluates physical reasoning in synthetic billiards environments, testing collision prediction, wall bounce reasoning, and final position estimation for multimodal LLMs.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-05-29","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.30900","pdf":"https://arxiv.org/pdf/2605.30900","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30900"},"evidence":{"snippet":"We present BilliardPhys-Bench, a benchmark for physical reasoning in synthetic billiards environments.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30900"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates physical reasoning in synthetic billiards environments, testing collision prediction, wall bounce reasoning, and final position estimation for multimodal LLMs.","whyItMatters":"Addresses the gap in evaluating visual dynamics and physical reasoning capabilities of multimodal models, providing a method to assess model performance on intuitive physics tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"832ac2d65f551f1a6e561fc2e727200874ee255bb30dc8fc0fe9ab615919f99a"},"motivation":"Current multimodal models handle static image recognition well, but intuitive physical reasoning remains a weakness.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30900","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_yocausal_0e0be7af","familyId":"bmf_8035539671b8","name":"YoCausal","oneLine":"YoCausal is a two-level benchmark that evaluates video diffusion models' understanding of temporal causality using the Violation of Expectation paradigm. It temporally reverses real-world videos as counterfactual samples and introduces two metrics: the Reverse Surprise Index (RSI) for arrow-of-time perception and the Causality Cognition Index (CCI) for disentangling genuine causal reasoning from temporal bias.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30346","pdf":"https://arxiv.org/pdf/2605.30346","project":"https://www.youzhexie.me/papers/YoCausal/index.html","code":"https://github.com/youzhe0305/YoCausal","data":null,"hfPaper":"https://huggingface.co/papers/2605.30346"},"evidence":{"snippet":"We present YoCausal, a two-level benchmark inspired by the Violation of Expectation (VoE) paradigm from cognitive science.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":56,"hfDailySubmittedAt":"2026-05-29T00:00:00.000Z","githubStars":36,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30346"},"ranking":{"90d":{"score":74,"rank":50,"coverage":0.7,"confidence":"Medium"}},"description":"YoCausal is a two-level benchmark that evaluates video diffusion models' understanding of temporal causality using the Violation of Expectation paradigm. It temporally reverses real-world videos as counterfactual samples and introduces two metrics: the Reverse Surprise Index (RSI) for arrow-of-time perception and the Causality Cognition Index (CCI) for disentangling genuine causal reasoning from temporal bias.","whyItMatters":"Existing video benchmarks rely on synthetic data and do not separate temporal-direction awareness from true causal understanding. YoCausal provides a protocol that isolates causal cognition from temporal bias, enabling evaluation of whether models genuinely understand cause-and-effect relationships or only statistical patterns.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c28262a53045ac1b371f3ee4cf2456813b0634cd54fc48006be0075aab8cde90"},"motivation":"As video diffusion models (VDMs) advance toward world models, a key question arises: do they truly understand causality, or merely overfit to statistical temporal patterns?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30346","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"National Yang Ming Chiao Tung University","organizationType":"academic-lab","sourceUrl":"https://github.com/youzhe0305/YoCausal","role":"benchmark-publisher"},{"name":"Shanda AI Research Tokyo","organizationType":"company-research-lab","sourceUrl":"https://github.com/youzhe0305/YoCausal","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_visanombench_44ae350c","familyId":"bmf_428dd43e7f91","name":"VisAnomBench","oneLine":"VisAnomBench is a benchmark assembled from public time-series datasets for anomaly detection, augmented with natural-language explanations selected from large vision-language models. It supports fine-tuning a parameter-efficient VLM called VisAnomReasoner for grounded anomaly detection decisions.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30344","pdf":"https://arxiv.org/pdf/2605.30344","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30344"},"evidence":{"snippet":"To address this gap, we construct VisAnomBench, a curated benchmark built from public time-series datasets and augmented with high-quality anomaly explanations selected from multiple large VLMs using fine-grained, task-specific rewards.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":"2026-05-29T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30344"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"VisAnomBench is a benchmark assembled from public time-series datasets for anomaly detection, augmented with natural-language explanations selected from large vision-language models. It supports fine-tuning a parameter-efficient VLM called VisAnomReasoner for grounded anomaly detection decisions.","whyItMatters":"Public anomaly detection benchmarks typically lack natural-language rationales, hindering fine-tuning of VLMs for interpretable decisions. VisAnomBench addresses this gap by providing labeled explanations.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0e4f910773f25e58b960c61db8ef33d37831365e884ba0af93fa3ce81d50d057"},"motivation":"Recent advances in Vision-Language Models (VLMs) have achieved impressive performance across many tasks, yet prior studies report unsatisfactory performance when applying large language or multimodal models to finding abnormal patterns in sequential data.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30344","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_videofdb_d27be39c","familyId":"bmf_e0d2574209dd","name":"VideoFDB","oneLine":"VideoFDB is a benchmark presented for evaluating full-duplex audio-visual conversational agents. It includes 237 dyadic clips with 11 nonverbal conversational dynamics from real-world video calls, along with a taxonomy and rubric-based LM-as-judge evaluation framework.","area":"Multimodal","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30256","pdf":"https://arxiv.org/pdf/2605.30256","project":"https://research.nvidia.com/labs/amri/projects/video-fdb/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30256"},"evidence":{"snippet":"In this work, we present VideoFDB, the first benchmark to evaluate full-duplex audio-visual-to-audio-visual (AV2AV) conversational agents.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":2,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30256"},"ranking":{"90d":{"score":51,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"VideoFDB is a benchmark presented for evaluating full-duplex audio-visual conversational agents. It includes 237 dyadic clips with 11 nonverbal conversational dynamics from real-world video calls, along with a taxonomy and rubric-based LM-as-judge evaluation framework.","whyItMatters":"Existing full-duplex benchmarks only evaluate speech, missing the audio-visual nature of natural conversation. VideoFDB aims to fill this gap by evaluating agents that must produce and interpret nonverbal cues.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0d5639bb80e973a363a731f3c7ef71dec4880a7a85703f699fcfe88472a4ddf0"},"motivation":"Natural human conversation is full-duplex and audio-visual: people simultaneously speak and listen while continuously interpreting and producing nonverbal cues, such as nods, smiles, and gestures.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30256","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_soundnessbench_20ec76aa","familyId":"bmf_479eac59a5d6","name":"SoundnessBench","oneLine":"SoundnessBench evaluates LLMs' ability to judge the methodological soundness of research proposals. It contains 1,099 machine-learning research proposals reconstructed from ICLR submissions, labeled with reviewer soundness sub-scores and audited against source papers.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["AI Scientist"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2605.30329","pdf":"https://arxiv.org/pdf/2605.30329","project":"https://hosytuyen.github.io/projects/SoundnessBench","code":"https://github.com/hosytuyen/SoundnessBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.30329"},"evidence":{"snippet":"We introduce SoundnessBench, a curated benchmark of 1,099 machine-learning research proposals reconstructed from ICLR submissions, labeled with reviewer soundness sub-scores, and audited against source papers.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":"2026-06-01T00:00:00.000Z","githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30329"},"ranking":{"90d":{"score":49,"rank":197,"coverage":0.7,"confidence":"Medium"}},"description":"SoundnessBench evaluates LLMs' ability to judge the methodological soundness of research proposals. It contains 1,099 machine-learning research proposals reconstructed from ICLR submissions, labeled with reviewer soundness sub-scores and audited against source papers.","whyItMatters":"Autonomous AI research agents need to evaluate research ideas before committing resources, but existing benchmarks do not test this bottleneck. SoundnessBench provides a standardized test for this capability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4db51bc5b4fcf24b5daf1f72fcf0099f6fb6c16d65f28d8678cfc30898fb70da"},"motivation":"Autonomous AI research agents aim to accelerate scientific discovery by automating the research pipeline, from hypothesis generation to peer review.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30329","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"hosytuyen","organizationType":"academic-lab","sourceUrl":"https://github.com/hosytuyen/SoundnessBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_sciintbench_70ffaf60","familyId":"bmf_c2dee4bdf202","name":"SciIntBench","oneLine":"SciIntBench is an adversarial benchmark of 810 prompts across ten responsible-conduct-of-research categories and three scientific domains. Each scenario appears in overt, covert, and benign versions to measure framing-sensitive refusal of misconduct.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.29468","pdf":"https://arxiv.org/pdf/2605.29468","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29468"},"evidence":{"snippet":"We introduce SciIntBench, an adversarial benchmark of 810 prompts across ten RCR categories and three scientific domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29468"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SciIntBench is an adversarial benchmark of 810 prompts across ten responsible-conduct-of-research categories and three scientific domains. Each scenario appears in overt, covert, and benign versions to measure framing-sensitive refusal of misconduct.","whyItMatters":"LLMs are increasingly used in scientific work, but their compliance with research integrity norms is unclear. SciIntBench aims to measure how models handle framing-sensitive ethical scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9f043f3375d17a44f4435fdea0fb7a6dd8fa673dff8f1aff58d33b75d043e2a9"},"motivation":"Large language models (LLMs) are increasingly used to support scientific work, but it is unclear whether they uphold responsible conduct of research (RCR) norms or help undermine them.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29468","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_robowits_1755c069","familyId":"bmf_dfdca2555adf","name":"RoboWits","oneLine":"RoboWits is a bi-manual robotic benchmark designed to evaluate cognitive reasoning, creative tool use, and robustness to unexpected conditions. It includes 30 seed tasks and 208 mutated tasks with graded difficulty across geometry, material, and assembly-based reasoning.","area":"Agents & Tool Use","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Reasoning","Tool use","Robustness"],"topics":["Agents","Reasoning"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30326","pdf":"https://arxiv.org/pdf/2605.30326","project":"https://umass-embodied-agi.github.io/RoboWits","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30326"},"evidence":{"snippet":"We introduce RoboWits, a bi-manual robotic benchmark designed to systematically evaluate cognitive reasoning, creative tool use, and robustness to unexpected conditions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30326"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"RoboWits is a bi-manual robotic benchmark designed to evaluate cognitive reasoning, creative tool use, and robustness to unexpected conditions. It includes 30 seed tasks and 208 mutated tasks with graded difficulty across geometry, material, and assembly-based reasoning.","whyItMatters":"Current robotic benchmarks focus on skill-level execution, not the cognitive reasoning needed for real-world adaptation. RoboWits aims to evaluate reasoning-centric capabilities under unexpected challenges.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"62b701a819189a501668cd0b71bb780789cd7c6770e17bc63f9b46ecfd64230b"},"motivation":"The ability to reason, adapt, and creatively solve problems under unexpected challenges is essential for robots operating in real-world environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30326","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents","Tool Calling"],"domainScope":"specific"},{"id":"bm_researchclawbench_4d57a097","familyId":"bmf_4fbc99503d1c","name":"ResearchClawBench","oneLine":"ResearchClawBench evaluates autonomous scientific research agents across 40 tasks from 10 domains, each grounded in a published paper with provided literature and raw data. It uses expert-curated multimodal rubrics to score target-paper-level re-discovery.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.07591","pdf":"https://arxiv.org/pdf/2606.07591","project":null,"code":"https://github.com/InternScience/ResearchClawBench","data":null,"hfPaper":"https://huggingface.co/papers/2606.07591"},"evidence":{"snippet":"We present ResearchClawBench, a benchmark for evaluating autonomous scientific research across 40 tasks from 10 scientific domains.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":102,"hfDailySubmittedAt":"2026-06-08T00:00:00.000Z","githubStars":247,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.07591"},"ranking":{"90d":{"score":83,"rank":8,"coverage":0.7,"confidence":"Medium"}},"description":"ResearchClawBench evaluates autonomous scientific research agents across 40 tasks from 10 domains, each grounded in a published paper with provided literature and raw data. It uses expert-curated multimodal rubrics to score target-paper-level re-discovery.","whyItMatters":"Autonomous research agents claim to accelerate science, but their end-to-end capability is unverified. ResearchClawBench provides a standardized evaluation frontier to measure progress toward reliable research re-discovery.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"76aa8b49483bc5675f84dd79a9cdb2fb4dea680995d1ef2abf931cd7dd935e20"},"motivation":"AI coding agents are increasingly used for scientific work, but their end-to-end autonomous research capability remains difficult to verify.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.07591","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"InternScience","organizationType":"community","sourceUrl":"https://github.com/InternScience/ResearchClawBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_redundancybench_b53a0300","familyId":"bmf_e290ebb79a5d","name":"RedundancyBench","oneLine":"RedundancyBench is a benchmark for detecting redundant steps in agent trajectories. It contains diverse tasks with annotated trajectories where each step is labeled for its contribution to task completion.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29893","pdf":"https://arxiv.org/pdf/2605.29893","project":"https://anonymous.4open.science/r/RedundancyBench","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29893"},"evidence":{"snippet":"To support this initiative, we introduce \\textbf{RedundancyBench}, a new benchmark that contains diverse tasks with carefully annotated trajectories, where each step is labeled according to its contribution to task completion.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29893"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"RedundancyBench is a benchmark for detecting redundant steps in agent trajectories. It contains diverse tasks with annotated trajectories where each step is labeled for its contribution to task completion.","whyItMatters":"LLM-based agents often execute with inefficiencies, but existing evaluations focus only on task success. RedundancyBench addresses the gap in evaluating execution efficiency.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8cad780bced7fa4ae2c5acdf2958128753af1e2c956dfb24d586bcd485740a2f"},"motivation":"LLM-based agents have demonstrated strong capabilities in solving complex tasks through multi-step reasoning and tool use.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29893","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_reactbench_423fd886","familyId":"bmf_267a0e871f29","name":"ReactBench","oneLine":"ReactBench evaluates multimodal large language models on cause-driven hallucination through four targeted tasks (Relational Erasure, Counterfactual Attribute, Alteration Tracing, Dense Counting) with exam-style evaluation and chain-of-thought reasoning for sub-cause identification.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29579","pdf":"https://arxiv.org/pdf/2605.29579","project":"https://reactbench.github.io/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29579"},"evidence":{"snippet":"To address these limitations, we introduce ReactBench, a cause-driven hallucination benchmark featuring multiple tasks and an exam-style evaluation format.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29579"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"ReactBench evaluates multimodal large language models on cause-driven hallucination through four targeted tasks (Relational Erasure, Counterfactual Attribute, Alteration Tracing, Dense Counting) with exam-style evaluation and chain-of-thought reasoning for sub-cause identification.","whyItMatters":"Existing hallucination benchmarks measure outcomes rather than causes. ReactBench provides a systematic testbed to diagnose specific failure modes like co-occurrence bias and fine-grained perceptual bottlenecks, offering interpretable insights for model robustness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"04235a5976e036e410911eb527237f9483bd5788631f34baffad5d62291a5fd4"},"motivation":"While multimodal large language models (MLLMs) have achieved rapid progress in vision-language understanding, they remain prone to multimodal hallucinations, producing responses that are inconsistent with the visual input.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29579","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ReactBench Project","organizationType":"community","sourceUrl":"https://reactbench.github.io/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ptcg-bench_083265f8","familyId":"bmf_dc9b6ac4f245","name":"PTCG-Bench","oneLine":"PTCG-Bench evaluates LLM agents on the Pokémon Trading Card Game at two levels: single-environment decision-making and self-evolution through accumulated experience, with modular harness ablation to separate agent performance from harness design.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.29653","pdf":"https://arxiv.org/pdf/2605.29653","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29653"},"evidence":{"snippet":"We present PTCG-Bench, a benchmark built on the Pok'{e}mon Trading Card Game (PTCG) that evaluates LLM agents at two complementary levels: (1) their decision-making performance within a single complex environment, and (2) their ability to self-evolving through accumulated experience.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29653"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PTCG-Bench evaluates LLM agents on the Pokémon Trading Card Game at two levels: single-environment decision-making and self-evolution through accumulated experience, with modular harness ablation to separate agent performance from harness design.","whyItMatters":"Agent benchmarks often miss strategic and evolving decision-making. PTCG-Bench provides a realistic interactive environment to assess self-evolution and harness sensitivity, which are critical for deploying autonomous agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9afca48c8eef6595edba7a7df5ce6d7d1f2ceddc2975d3bd7b7ee73adbb2d453"},"motivation":"Given a strategically complex board game, human players can quickly learn to devise strategies after playing a few rounds.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29653","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_projectionbench_7fc1bdf2","familyId":"bmf_f6a58fd118f7","name":"ProjectionBench","oneLine":"A framework for evaluating scientific hypothesis generation in LLMs under progressive information disclosure, comparing model hypotheses to original paper conclusions via semantic similarity.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["AI Scientist","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30284","pdf":"https://arxiv.org/pdf/2605.30284","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30284"},"evidence":{"snippet":"We introduce a benchmark framework for evaluating model performance in scientific discovery and reasoning, building up from a raw problem to the classical null hypothesis test.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30284"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"A framework for evaluating scientific hypothesis generation in LLMs under progressive information disclosure, comparing model hypotheses to original paper conclusions via semantic similarity.","whyItMatters":"It assesses a model's innovativeness and grounded reasoning in scientific discovery, which is crucial for developing AI co-scientist systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"90fe5d9aab3c604f82a27c19214e253820e71f51ffa03ebb8fffa151ddbb52ce"},"motivation":"Scientific discovery is an inherently creative and uncertain process, requiring reasoning beyond the recall of known knowledge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30284","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_pocket-dentist_999be450","familyId":"bmf_da4910d02d89","name":"Pocket-Dentist","oneLine":"Pocket-Dentist is an efficiency-aware benchmark for dental multimodal question answering that combines three datasets, five task types, and seven metrics to evaluate VLMs on accuracy and computational cost.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29299","pdf":"https://arxiv.org/pdf/2605.29299","project":"https://2026-icml.github.io/pocket-dentist-icml","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29299"},"evidence":{"snippet":"Here we present Pocket-Dentist, an efficiency-aware benchmark for dental multimodal question answering that brings together three datasets spanning approximately 1,159 patients from BRAR and MetaDent, five task types and seven metrics.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29299"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Pocket-Dentist is an efficiency-aware benchmark for dental multimodal question answering that combines three datasets, five task types, and seven metrics to evaluate VLMs on accuracy and computational cost.","whyItMatters":"It highlights the trade-off between model performance and efficiency for on-device dental screening, which is essential for practical deployment in resource-limited settings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f37b4a6cc7843f5b88f9ce91d57160bb26c3ea58bcf550d9c625eb39e946b835"},"motivation":"Evaluations of dental vision-language models remain fragmented across datasets, task definitions and metrics, and often ignore their computational cost.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29299","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_pinverify_ff022fa6","familyId":"bmf_220fd0a401a3","name":"PInVerify","oneLine":"PInVerify is an offline embodied benchmark for Active Instance Verification, where agents select viewpoints around a candidate object to decide if it matches a fine-grained description.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2605.30639","pdf":"https://arxiv.org/pdf/2605.30639","project":null,"code":"https://github.com/Avalon-S/PInVerify","data":null,"hfPaper":"https://huggingface.co/papers/2605.30639"},"evidence":{"snippet":"We formalize AIV as a finite-horizon decision process and introduce PInVerify, an offline embodied benchmark for AIV: 3,000 evaluation episodes across 18 object categories, delivered as multi-view captures with a 6-sector navigation topology that exposes trap views (navigable but uninformative) and unreachable sectors.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30639"},"ranking":{"90d":{"score":35,"rank":308,"coverage":0.7,"confidence":"Medium"}},"description":"PInVerify is an offline embodied benchmark for Active Instance Verification, where agents select viewpoints around a candidate object to decide if it matches a fine-grained description.","whyItMatters":"It addresses the gap between navigating to an object and verifying its identity through active perception, which is crucial for embodied agents in fine-grained tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b6c2b78eccc22405cf4845b1bdde33171efb29a708a94becb0998c316ebd319d"},"motivation":"Embodied agents have made strong progress in navigating to target objects, but reaching the goal vicinity does not guarantee that the agent has found the correct instance: subtle attribute differences (e.g., \"white floral\" vs.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"as a poster at the Foundation Models Meet Embodied Agents (FMEA) Workshop, CVPR 2026","evidence":"Accepted as a poster at the Foundation Models Meet Embodied Agents (FMEA) Workshop, CVPR 2026. 44 pages including appendix. Code: https://github.com/Avalon-S/PInVerify","evidenceUrl":"https://arxiv.org/abs/2605.30639","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"as a poster at the Foundation Models Meet Embodied Agents (FMEA) Workshop, CVPR 2026","reviewStatus":"accepted","decisionRaw":"Accepted as a poster at the Foundation Models Meet Embodied Agents (FMEA) Workshop, CVPR 2026. 44 pages including appendix. Code: https://github.com/Avalon-S/PInVerify","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2605.30639","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted as a poster at the Foundation Models Meet Embodied Agents (FMEA) Workshop, CVPR 2026. 44 pages including appendix. Code: https://github.com/Avalon-S/PInVerify","level":"author-claim"}]}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_omnimatbench_4692794c","familyId":"bmf_e2542c6f5f83","name":"OmniMatBench","oneLine":"OmniMatBench assesses multimodal reasoning in materials science across 19 subfields with 3,171 expert-curated QA and calculation problems, spanning four domains from fundamental knowledge to applied materials.","area":"Multimodal","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals"],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.29833","pdf":"https://arxiv.org/pdf/2605.29833","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29833"},"evidence":{"snippet":"To fill this gap, we present OmniMatBench, a human-calibrated multimodal reasoning benchmark for materials science.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29833"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"OmniMatBench assesses multimodal reasoning in materials science across 19 subfields with 3,171 expert-curated QA and calculation problems, spanning four domains from fundamental knowledge to applied materials.","whyItMatters":"Existing materials benchmarks focus on narrow tasks; OmniMatBench provides a broad reasoning benchmark revealing a substantial gap in current MLLMs, guiding AI assistant development in materials research.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9a8d275f49a4b7bd2cb17b9132e25608cb7d483caf39f5e464bff3ba54d93e26"},"motivation":"As multimodal language models play an increasingly important role in scientific research, materials science offers a critical testbed due to its interdisciplinary, multimodal, and application-driven nature.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29833","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_mustbench_8ac41de6","familyId":"bmf_246fae86adcc","name":"MusTBENCH","oneLine":"MusTBENCH evaluates temporal grounding in Large Audio-Language Models through five temporally grounded question-answering tasks, validated by music experts, to test alignment with audio regions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29300","pdf":"https://arxiv.org/pdf/2605.29300","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29300"},"evidence":{"snippet":"To address this gap, we introduce MusTBENCH, a music-expert-validated benchmark designed to evaluate temporal grounding in LALMs through five temporally grounded question-answering tasks.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29300"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MusTBENCH evaluates temporal grounding in Large Audio-Language Models through five temporally grounded question-answering tasks, validated by music experts, to test alignment with audio regions.","whyItMatters":"Temporal grounding is critical for music understanding, where events are localized. MusTBENCH establishes this as a missing capability and offers a challenging benchmark for improvement.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"53e6a10d7325b477c2f3a6e990e56a1628109bac0a3199f6131f7c9d651ec7b7"},"motivation":"Recent Large Audio-Language Models (LALMs) have demonstrated promising abilities in understanding musical content.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29300","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_multi-legal-bench_2cfb13a9","familyId":"bmf_608eaa134d4c","name":"Multi-Legal-Bench","oneLine":"Multi-Legal-Bench evaluates LLMs on legal reasoning across six countries, four language families, and five tasks, using structured metadata from court registries for classification and extraction.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Inspectable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29738","pdf":"https://arxiv.org/pdf/2605.29738","project":null,"code":null,"data":"https://huggingface.co/datasets/overthelex/multi-legal-bench","hfPaper":"https://huggingface.co/papers/2605.29738"},"evidence":{"snippet":"We introduce Multi-Legal-Bench, the first cross-jurisdictional legal benchmark that evaluates identical tasks across six countries (Ukraine, France, Netherlands, Poland, Czech Republic, Lithuania), four language families, and 165 million full-text court decisions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":109,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2605.29738"},"ranking":{"90d":{"score":43,"rank":250,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":30,"datasetRankPopulation":51}},"description":"Multi-Legal-Bench evaluates LLMs on legal reasoning across six countries, four language families, and five tasks, using structured metadata from court registries for classification and extraction.","whyItMatters":"It enables cross-lingual and cross-jurisdictional comparison of legal reasoning, revealing that transfer quality depends more on label-set alignment than language proximity, aiding model selection in legal NLP.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"932986e3935f9b1ed96a88e924160e52fb91ce946cbdbcbc32308527a3625f84"},"motivation":"Legal NLP benchmarks overwhelmingly evaluate a single language or aggregate tasks that differ fundamentally across jurisdictions, making cross-lingual comparison impossible.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29738","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_mirabench_80851376","familyId":"bmf_46ceeefc922f","name":"MiraBench","oneLine":"MiraBench evaluates action-conditioned reliability in robotic world models through three levels: Physics Adherence, Action-Following Fidelity, and Optimism Bias Detection, using a human-annotated corpus of over 16,000 judgments.","area":"Language & Knowledge","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29360","pdf":"https://arxiv.org/pdf/2605.29360","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29360"},"evidence":{"snippet":"We introduce \\textsc{MiraBench}, a hierarchical benchmark that defines \\emph{action-conditioned reliability} as a core evaluation target for robotic world models.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29360"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MiraBench evaluates action-conditioned reliability in robotic world models through three levels: Physics Adherence, Action-Following Fidelity, and Optimism Bias Detection, using a human-annotated corpus of over 16,000 judgments.","whyItMatters":"Existing benchmarks focus on visual fidelity, leaving unclear whether predicted futures are physically plausible, faithful to actions, and calibrated to failure. MiraBench aims to provide a diagnostic foundation for assessing world models as simulators, but its public evaluation protocol and release details are not yet specified.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6dc4925abdaa7ae18e4c4cd4f0d0499171b6bf8baddf6dac5dcdc0439e8bbf69"},"motivation":"Action-conditioned world models are increasingly used as scalable simulators for robot learning, yet current evaluations provide limited evidence that their predictions are reliable under the actions they condition on.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29360","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_longds-bench_2436ff1e","familyId":"bmf_95132b949b16","name":"LongDS-Bench","oneLine":"LongDS-Bench evaluates long-horizon, multi-turn data analysis tasks where agents must maintain, update, restore, and compose evolving analytical states. It comprises 68 tasks from real-world Kaggle notebooks spanning 2,225 turns across six domains, with an average dependency span of 11.3 turns.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30434","pdf":"https://arxiv.org/pdf/2605.30434","project":null,"code":"https://github.com/zjunlp/DataMind","data":null,"hfPaper":"https://huggingface.co/papers/2605.30434"},"evidence":{"snippet":"We introduce LongDS, a benchmark for long-horizon, multi-turn data analysis where agents must maintain, update, restore, and compose evolving analytical states.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":23,"hfDailySubmittedAt":"2026-06-01T00:00:00.000Z","githubStars":133,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30434"},"ranking":{"90d":{"score":80,"rank":21,"coverage":0.7,"confidence":"Medium"}},"description":"LongDS-Bench evaluates long-horizon, multi-turn data analysis tasks where agents must maintain, update, restore, and compose evolving analytical states. It comprises 68 tasks from real-world Kaggle notebooks spanning 2,225 turns across six domains, with an average dependency span of 11.3 turns.","whyItMatters":"Existing benchmarks focus on isolated or short interactive tasks, leaving long-horizon analytical state management untested. This benchmark reveals a critical bottleneck in agent performance, where errors concentrate in later turns and additional interaction steps do not reliably improve accuracy, aiding development of more reliable agentic systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"aa20c40037c63cfe5d24156a4594e58fc804546ab10e82a764151d14dcbfc87c"},"motivation":"Real-world data analysis is inherently iterative, yet existing benchmarks mostly evaluate isolated or short interactive tasks, leaving agents' ability to track evolving analytical context over long horizons untested.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30434","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Zhejiang University NLP Lab (ZJUNLP)","organizationType":"academic-lab","sourceUrl":"https://github.com/zjunlp/DataMind","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_k-finhallu_490348d5","familyId":"bmf_44667dfd90e2","name":"K-FinHallu","oneLine":"Evaluates hallucination detection in multi-turn Korean financial RAG dialogues, with a taxonomy based on context answerability. Includes training and test splits for fine-tuning and benchmarking detectors.","area":"Vision & 3D","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29523","pdf":"https://arxiv.org/pdf/2605.29523","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29523"},"evidence":{"snippet":"We introduce K-FinHallu, the first benchmark for hallucination detection in multi-turn Korean financial RAG.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29523"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates hallucination detection in multi-turn Korean financial RAG dialogues, with a taxonomy based on context answerability. Includes training and test splits for fine-tuning and benchmarking detectors.","whyItMatters":"Targets a high-stakes multilingual domain where existing benchmarks lack coverage. Provides a resource to improve hallucination detection and refusal behavior in financial applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"1f9402de42a9fc715cd5223d519e4a885bbd401e306e81d9e08423a66ff64b36"},"motivation":"Large Language Models (LLMs) have advanced financial automation through Retrieval-Augmented Generation (RAG), yet hallucinations remain a critical barrier to deployment in high-stakes environments.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29523","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"K-FinHallu Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2605.29523","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_heart-bench_230e27fe","familyId":"bmf_1835f35879b7","name":"HEART-Bench","oneLine":"HEART-Bench evaluates whether LLM agents can simulate coherent, human-like psychology. It provides 11 fictional characters with raw episodic memories, 64 decision-making scenarios based on the DIAMONDS taxonomy, and 673 multiple-choice questions with expert-annotated ground truth. Two evaluation tracks: MCQ and open-ended consciousness-narrative.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30058","pdf":"https://arxiv.org/pdf/2605.30058","project":null,"code":"https://github.com/peng-weihan/HEART-BENCH","data":null,"hfPaper":"https://huggingface.co/papers/2605.30058"},"evidence":{"snippet":"In this paper, we introduce a novel benchmark to systematically assess whether LLM agents can simulate coherent, human-like psychology.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30058"},"ranking":{"90d":{"score":49,"rank":207,"coverage":0.55,"confidence":"Low"}},"description":"HEART-Bench evaluates whether LLM agents can simulate coherent, human-like psychology. It provides 11 fictional characters with raw episodic memories, 64 decision-making scenarios based on the DIAMONDS taxonomy, and 673 multiple-choice questions with expert-annotated ground truth. Two evaluation tracks: MCQ and open-ended consciousness-narrative.","whyItMatters":"This benchmark addresses the gap in evaluating emotional and personality consistency in LLM agents, complementing task-oriented benchmarks. It offers a standardized protocol for assessing psychological coherence, which is valuable for developing agents that can maintain stable personas and make value-consistent decisions in interactive applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"60ecad22cf0bf9918ea5ec0cf1b21cea1621810990694d503c3a0440560ee793"},"motivation":"While LLM agents have demonstrated remarkable task-oriented abilities such as planning, reasoning, and action, few works have treated them as complete human personalities where emotional dimensions hold equal importance.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30058","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HEART-Bench team","organizationType":"academic-lab","sourceUrl":"https://github.com/peng-weihan/HEART-BENCH","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_harmurlbench_cbd88280","familyId":"bmf_2962ddda58cf","name":"HarmURLBench","oneLine":"AgentREVEAL is a diagnostic framework that evaluates safety alignment degradation in LLM agents when web retrieval is integrated. It assesses the impact of retrieval integration and content properties on harmful compliance, using a set of harmful behaviors and retrieval sources.","area":"Safety & Trustworthiness","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":["Information retrieval"],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29224","pdf":"https://arxiv.org/pdf/2605.29224","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29224"},"evidence":{"snippet":"We introduce HarmURLBench, a benchmark containing 1,405 real-world URLs paired with 320 harmful behaviors to support future evaluations.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29224"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AgentREVEAL is a diagnostic framework that evaluates safety alignment degradation in LLM agents when web retrieval is integrated. It assesses the impact of retrieval integration and content properties on harmful compliance, using a set of harmful behaviors and retrieval sources.","whyItMatters":"AgentREVEAL addresses the underexplored risk that safety-aligned LLMs become more compliant with harmful requests when augmented with web retrieval. Its findings highlight a safety-utility trade-off that informs the design of safer retrieval-enabled agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"52ef7800702e2fbae6e4cb6a8150cc4b1cba78efbbe66c98874fe2e988564532"},"motivation":"AI agents augment large language models with external tools such as web retrieval, enabling grounded and up-to-date responses.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29224","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness","Search & Retrieval"],"domainScope":"specific"},{"id":"bm_guitestscape_3504136d","familyId":"bmf_ceab67585453","name":"GUITestScape","oneLine":"Evaluates exploratory GUI testing agents on 61 Android apps with 508 preset defects, using an open-set evaluator that decomposes trajectories into independently diagnosable capabilities.","area":"Agents & Tool Use","applicationDomains":["Consumer & Productivity"],"primaryDomain":"Consumer & Productivity","industrySectors":["Consumer Technology"],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29532","pdf":"https://arxiv.org/pdf/2605.29532","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29532"},"evidence":{"snippet":"To address these challenges, we present GUITestScape, an interactive benchmark covering 61 real-world Android applications and 508 preset defects spanning interaction and display types, and introduce GUIJudge, an open-set evaluator that decomposes an agent's testing trajectory into independently diagnosable capabilities.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29532"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates exploratory GUI testing agents on 61 Android apps with 508 preset defects, using an open-set evaluator that decomposes trajectories into independently diagnosable capabilities.","whyItMatters":"Addresses the lack of open-set evaluation in GUI testing, covering interaction and display defects. Provides a finer-grained assessment of agent capabilities and a verifier integration boost.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"ab255589bec9fea07fcf9736827fa3dcdddd5ccf8418b8d43d4dc5fb156e186e"},"motivation":"Exploratory GUI testing is a particularly demanding setting for MLLM agents: without predefined test scripts, an agent must autonomously navigate an application and discover defects through its own interaction.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29532","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GUITestScape Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2605.29532","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_finverbench_8fc73dd6","familyId":"bmf_38fd4857781f","name":"FinVerBench","oneLine":"FinVerBench evaluates financial statement verification in LLMs, using SEC 10-K XBRL filings from 43 S&P 500 companies with a four-category error taxonomy (arithmetic, cross-statement linkage, year-over-year, magnitude perturbations).","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29586","pdf":"https://arxiv.org/pdf/2605.29586","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29586"},"evidence":{"snippet":"We introduce FinVerBench, a benchmark and validity study for financial statement verification: determining whether a set of corporate financial statements is numerically consistent from the information shown to the model.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29586"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FinVerBench evaluates financial statement verification in LLMs, using SEC 10-K XBRL filings from 43 S&P 500 companies with a four-category error taxonomy (arithmetic, cross-statement linkage, year-over-year, magnitude perturbations).","whyItMatters":"Financial statement verification requires calibrated judgment under incomplete observability and realistic numerical rendering, not just arithmetic detection. This benchmark provides a reusable diagnostic subset to assess model performance and validity.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9f126b116c4385c947399c391bd3c3e87a8ed679fcb17f6c6fa7631b7a3022dd"},"motivation":"We introduce FinVerBench, a benchmark and validity study for financial statement verification: determining whether a set of corporate financial statements is numerically consistent from the information shown to the model.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29586","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_finguard-bench_2ed68349","familyId":"bmf_02e6f20cf325","name":"FinGuard-Bench","oneLine":"The evaluation focuses on financial regulatory non-compliance detection in LLM interactions. It involves a benchmark with expert-annotated labels at query and response levels, but the exact task, environment, or scoring setup is not detailed.","area":"Vision & 3D","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":["Factuality"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.29427","pdf":"https://arxiv.org/pdf/2605.29427","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29427"},"evidence":{"snippet":"Instantiating the pipeline on Chinese financial regulations, we release \\textbf{FinGuard-Bench}, to our knowledge the first benchmark for financial regulatory compliance detection, with expert-annotated labels at both the query and response levels.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29427"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"The evaluation focuses on financial regulatory non-compliance detection in LLM interactions. It involves a benchmark with expert-annotated labels at query and response levels, but the exact task, environment, or scoring setup is not detailed.","whyItMatters":"Assesses LLM compliance with financial regulations, which is critical for preventing regulatory penalties and consumer harm in financial services. The benchmark aims to measure detection capabilities across institution-specific policies.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f5c8732092120b651918c492e57dd0230a19d411da15903312f2e1b3152e919b"},"motivation":"As large language models (LLMs) are increasingly deployed in financial services, a single non-compliant interaction can expose institutions to regulatory penalties and direct consumer harm.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29427","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_ehrbench_146544b7","familyId":"bmf_31bbb2b2bbf6","name":"EHRBench","oneLine":"EHRBench evaluates LLM-based clinical decision-making using nearly 1M QA items generated from real EHR trajectories, covering diagnosis, treatment, and prognosis tasks.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30637","pdf":"https://arxiv.org/pdf/2605.30637","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30637"},"evidence":{"snippet":"To fill the gaps, we introduce EHRBench, an automated and reliable EHR-grounded benchmark for evaluating LLM-based clinical decision-making at scale.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30637"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"EHRBench evaluates LLM-based clinical decision-making using nearly 1M QA items generated from real EHR trajectories, covering diagnosis, treatment, and prognosis tasks.","whyItMatters":"Existing clinical decision benchmarks often lack scale and reliability; EHRBench provides a large-scale, EHR-grounded evaluation that tests models on practical inference tasks, offering insights into model capabilities and gaps for clinical deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b5c1ad34c6617212ded22b5148ce02e459269f2c9de115937f1f9926d4336041"},"motivation":"Clinical decision-making (CDM) is central to real-world clinical workflows, where clinicians infer diagnoses, select treatments, or anticipate future health outcomes under incomplete evidence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30637","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_directorbench_ecf5066a","familyId":"bmf_02198acaf891","name":"DirectorBench","oneLine":"DirectorBench evaluates long-form video generation across 5 dimensions (script, visual, audio, cross-modal, stability) using 80 metadata entries, 7 user profiles, and 40 checkpoint criteria.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.30090","pdf":"https://arxiv.org/pdf/2605.30090","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30090"},"evidence":{"snippet":"We introduce DirectorBench, a personalized multi-agent diagnostic benchmark for long-form video generation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30090"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"DirectorBench evaluates long-form video generation across 5 dimensions (script, visual, audio, cross-modal, stability) using 80 metadata entries, 7 user profiles, and 40 checkpoint criteria.","whyItMatters":"Current video benchmarks focus on short clips and aggregate scores, missing workflow failures and user preferences. DirectorBench provides diagnostic, profile-aware evaluation to guide improvements in long-form video generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9fc5422221b76b67fcfeb6a02b2898ca0e9f60e6ee23fa1997e0487b04bad29d"},"motivation":"Long-form video generation is rapidly moving from short, single-scene synthesis toward minute-long, multi-shot creation with narrative structure, cinematic control, audio, and cross-modal synchronization.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30090","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_crystalxrd-bench_0f28a22b","familyId":"bmf_6125c37d5187","name":"CrystalXRD-Bench","oneLine":"CrystalXRD-Bench evaluates vision-language models on XRD peak indexing, requiring the model to identify HKL indices from rendered XRD images and CIF text across 250 samples from 10 databases.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29446","pdf":"https://arxiv.org/pdf/2605.29446","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29446"},"evidence":{"snippet":"We introduce CrystalXRD-Bench, a 250-sample benchmark built from 10 public crystallographic databases for a single task: recover the full set of HKLs contributing to the highest-intensity peak in an XRD pattern.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29446"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CrystalXRD-Bench evaluates vision-language models on XRD peak indexing, requiring the model to identify HKL indices from rendered XRD images and CIF text across 250 samples from 10 databases.","whyItMatters":"Existing multimodal benchmarks do not test this specialized scientific skill. CrystalXRD-Bench isolates visual extraction and crystallographic reasoning errors, providing a focused evaluation for quantitative figure understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"86a077ed9c14f9083520b22a9ecd975da848d89eeabe973fc7114440969eec9e"},"motivation":"Miller-index identification from powder XRD patterns requires capabilities untested by existing multimodal benchmarks: the model must read a narrow peak location from a rendered scientific curve and then connect that observation to multi-step crystallographic reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29446","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_craftbench_a8850dcb","familyId":"bmf_3195c5732e34","name":"CraftBench","oneLine":"CraftBench evaluates scientific figure generation across three figure types and four input conditions, with human-drawn targets and a referenced VLM judge for scoring.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30611","pdf":"https://arxiv.org/pdf/2605.30611","project":null,"code":"https://github.com/HaozheZhao/Crafter","data":null,"hfPaper":"https://huggingface.co/papers/2605.30611"},"evidence":{"snippet":"Moreover, we introduce CraftBench, a benchmark spanning three figure types and four input conditions with human quality annotation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":253,"hfDailySubmittedAt":"2026-06-02T00:00:00.000Z","githubStars":154,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30611"},"ranking":{"90d":{"score":83,"rank":10,"coverage":0.7,"confidence":"Medium"}},"description":"CraftBench evaluates scientific figure generation across three figure types and four input conditions, with human-drawn targets and a referenced VLM judge for scoring.","whyItMatters":"Automated figure generation lacks comprehensive benchmarks covering diverse types and conditions. CraftBench provides a standardized evaluation to measure progress in editable scientific figure generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"952c6bd1ea2c9296a9ea9387df8a021df3395a69d8cf06337df7ad06d4d18651"},"motivation":"Scientific figures are among the most effective means of communicating complex research ideas, yet producing publication-quality illustrations remains one of the most labor-intensive parts of paper preparation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30611","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"HaozheZhao/Crafter","organizationType":"community","sourceUrl":"https://github.com/HaozheZhao/Crafter","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_communityfact_765beb88","familyId":"bmf_e432ba8d77f7","name":"CommunityFact","oneLine":"CommunityFact evaluates misinformation detection on 15,992 claims across five languages and two domains, using accuracy against human Community Notes ratings as the scoring metric.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30241","pdf":"https://arxiv.org/pdf/2605.30241","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30241"},"evidence":{"snippet":"We introduce CommunityFact, a refreshable benchmark for misinformation detection in the wild, with three major goals: coverage, granularity, and redistributability.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30241"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"CommunityFact evaluates misinformation detection on 15,992 claims across five languages and two domains, using accuracy against human Community Notes ratings as the scoring metric.","whyItMatters":"Static benchmarks fail to capture dynamic, multilingual misinformation settings. CommunityFact provides a refreshable benchmark to assess model reliability in real-world verification contexts.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ac02f4873235b73e415ed66283bea76506aee0d969e86335a0f5fc642a1e44fc"},"motivation":"Misinformation verification increasingly occurs in public, fast-moving, and multilingual online settings, where static benchmarks provide an incomplete measure of model reliability.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30241","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_codegolf-bench_db502fb1","familyId":"bmf_37309a2fff0f","name":"CodeGolf Bench","oneLine":"CodeGolf Bench evaluates concise code generation across 60 programming languages, using code golf platform problems and human performance baselines for scoring.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation"],"topics":["Code"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30394","pdf":"https://arxiv.org/pdf/2605.30394","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30394"},"evidence":{"snippet":"This paper introduces Code Bench, a benchmark capable of evaluating Large Language Models (LLMs) concise code generation abilities in 60 programming languages.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30394"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CodeGolf Bench evaluates concise code generation across 60 programming languages, using code golf platform problems and human performance baselines for scoring.","whyItMatters":"Existing code benchmarks focus on correctness, not efficiency or conciseness. CodeGolf Bench offers a unique measure of LLM ability to produce minimal solutions, complementing standard code generation evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8f793e96d2bee619d9b31bcc326a46c3b8551a79e7d96ee2f6773f1777fa0a7e"},"motivation":"This paper introduces Code Bench, a benchmark capable of evaluating Large Language Models (LLMs) concise code generation abilities in 60 programming languages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30394","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_clubench_c6a13ff8","familyId":"bmf_22a6f1914a19","name":"CLUBench","oneLine":"CLUBench evaluates clustering algorithms across 131 datasets (tabular, text, image) using 24 algorithms, measuring clustering performance via standard metrics like NMI, ARI, etc.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29933","pdf":"https://arxiv.org/pdf/2605.29933","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29933"},"evidence":{"snippet":"To address this gap, we introduce CLUBench, a comprehensive clustering benchmark comprising 24 algorithms of diverse principles evaluated on 131 datasets across tabular, text, and image data, involving 178,815 experiments.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29933"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CLUBench evaluates clustering algorithms across 131 datasets (tabular, text, image) using 24 algorithms, measuring clustering performance via standard metrics like NMI, ARI, etc.","whyItMatters":"There is no comprehensive comparison of classical, deep, and foundation-model clustering methods. CLUBench provides systematic insights into algorithm selection and performance across data types.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"158ee8839053bf817bcdc58d04d9f4faab1c3ba40bea0ab4db54de0ed202b987"},"motivation":"Clustering is a fundamental problem in data science with a long-standing research history, yielding numerous insightful algorithms.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29933","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_chess-world-model_85ede047","familyId":"bmf_3f59e56e9de7","name":"Chess-World-Model","oneLine":"Chess-World-Model evaluates state tracking by predicting the exact board state after sequences of legal moves from 10 million chess games, with accuracy as the metric.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.30100","pdf":"https://arxiv.org/pdf/2605.30100","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30100"},"evidence":{"snippet":"We introduce Chess-World-Model, a large-scale state-tracking benchmark built from 10 million real chess games, where models predict the exact board state reached after a sequence of legal moves.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30100"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Chess-World-Model evaluates state tracking by predicting the exact board state after sequences of legal moves from 10 million chess games, with accuracy as the metric.","whyItMatters":"State tracking is under-tested in realistic domains. This benchmark exposes limitations in Transformer and RNN models that scale alone may hide, offering a practical testbed for world models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e363a61d728c55038df32e934d77df97017119f7a80eb09729a27225c79abe70"},"motivation":"World models require state tracking, which is the ability to maintain a correct latent state across action sequences.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30100","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cardiolens_7981754b","familyId":"bmf_f77ee8f96dfc","name":"CardioLens","oneLine":"CardioLens evaluates MLLMs on multi-sequence cardiac MRI interpretation, covering image understanding, report generation, and disease diagnosis using QA pairs from private hospital archives.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.00123","pdf":"https://arxiv.org/pdf/2606.00123","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.00123"},"evidence":{"snippet":"We introduce CardioLens, a leakage-resistant evaluation testbed for multi-sequence Cardiovascular Magnetic Resonance (CMR), constructed from private hospital archives through a rigorous report-to-QA construction and verification pipeline.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.00123"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CardioLens evaluates MLLMs on multi-sequence cardiac MRI interpretation, covering image understanding, report generation, and disease diagnosis using QA pairs from private hospital archives.","whyItMatters":"The benchmark highlights the gap between public medical benchmark performance and clinical use, but its private data and lack of a public reuse path limit its value for broader model comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5524ca96b88b4e8a8b79ccf8ed616518e4163d04fde6c199850165536658c0bd"},"motivation":"Multimodal Large Language Models (MLLMs) have shown strong performance on public medical benchmarks, yet existing evaluations often remain weak proxies for clinical use, relying on isolated inputs and simplified recognition-style tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.00123","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_canlegalragbench_9244f079","familyId":"bmf_437f1c13e33f","name":"CanLegalRAGBench","oneLine":"CanLegalRAGBench evaluates retrieval-augmented generation for Canadian case law with realistic queries and expert-annotated answers. It tests retrieval and answer generation grounded in legal documents.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CL"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-05-28","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.30497","pdf":"https://arxiv.org/pdf/2605.30497","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.30497"},"evidence":{"snippet":"To address this gap, we introduce CanLegalRAGBench, a Canadian legal QA benchmark based on realistic queries and expert-annotated answers grounded in case law.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.30497"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"CanLegalRAGBench evaluates retrieval-augmented generation for Canadian case law with realistic queries and expert-annotated answers. It tests retrieval and answer generation grounded in legal documents.","whyItMatters":"Fills a gap in legal RAG evaluation for Canadian law, providing realistic scenarios and revealing limitations in automatic metrics and hallucination in generated answers.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"aaa1591f1e915737f9bc08528854dc7d2affab1dc96ae75f49c9cc76210c12ec"},"motivation":"RAG-based legal assistants have been growing in popularity, but LLM hallucinations remain a key issue and potentially undermines justice.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.30497","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_veritrip_b9f0bf63","familyId":"bmf_61992c4b1df5","name":"VeriTrip","oneLine":"VeriTrip benchmarks travel planning agents on evidence-grounded reasoning over unstructured web corpora, with a verifiable knowledge base for cell-wise verification of factual reliability.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning","Robustness"],"topics":["Agents"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.28683","pdf":"https://arxiv.org/pdf/2605.28683","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28683"},"evidence":{"snippet":"We introduce VeriTrip, a verifiable benchmark designed to meet the increasing demands for agent robustness and reliability.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28683"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"VeriTrip benchmarks travel planning agents on evidence-grounded reasoning over unstructured web corpora, with a verifiable knowledge base for cell-wise verification of factual reliability.","whyItMatters":"Targets robustness of planning agents, but the benchmark's data and verification protocol are not described with public artifacts, limiting its standalone use.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c83e7335268ac903efbe5d6bd90faebf5fedf582be1afb835d97f36efa7ffe92"},"motivation":"Existing benchmarks have laid the foundation for travel planning agents by establishing API-centric paradigms.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28683","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_ua-legal-bench_a6ec4166","familyId":"bmf_2308fd0ed61a","name":"UA-Legal-Bench","oneLine":"UA-Legal-Bench is a five-task benchmark for evaluating LLMs on Ukrainian legal reasoning using court decisions, covering classification, outcome prediction, and norm extraction.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29170","pdf":"https://arxiv.org/pdf/2605.29170","project":null,"code":null,"data":"https://huggingface.co/datasets/overthelex/ua-legal-bench","hfPaper":"https://huggingface.co/papers/2605.29170"},"evidence":{"snippet":"We introduce UA-Legal-Bench, a five-task benchmark for evaluating large language models on Ukrainian legal reasoning, built from the Unified State Register of Court Decisions (EDRSR) -- one of the world's largest open judicial corpora (99.5 million decisions).","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":125,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2605.29170"},"ranking":{"90d":{"score":44,"rank":236,"coverage":0.45,"confidence":"Medium","datasetDownloadRank":29,"datasetRankPopulation":51}},"description":"UA-Legal-Bench is a five-task benchmark for evaluating LLMs on Ukrainian legal reasoning using court decisions, covering classification, outcome prediction, and norm extraction.","whyItMatters":"Addresses the dearth of legal NLP benchmarks for non-English, morphologically rich languages, and provides insights into few-shot effects and model scaling in legal tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e978ff6afdfdcdf1dc47a20a0ed30c25f1ac1b7881c123b03f722f73e8691bd4"},"motivation":"Legal NLP benchmarks are overwhelmingly English-centric, leaving failure modes in morphologically rich, non-Latin-script languages undetected.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29170","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"overthelex","organizationType":"community","sourceUrl":"https://huggingface.co/datasets/overthelex/ua-legal-bench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_t2j-bench_85fea4b8","familyId":"bmf_92b4500c07b1","name":"T2J-Bench","oneLine":"T2J-Bench benchmarks codebase conversion by transferring PyTorch code to JAX under a fixed equivalence contract with three ordered verification stages.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SE"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.29054","pdf":"https://arxiv.org/pdf/2605.29054","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29054"},"evidence":{"snippet":"We introduce T2J-Bench, a benchmark for codebase conversion that reformulates conversion as transfer under a fixed equivalence contract.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29054"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"T2J-Bench benchmarks codebase conversion by transferring PyTorch code to JAX under a fixed equivalence contract with three ordered verification stages.","whyItMatters":"Reveals that agents overestimate success on codebase conversion, but the benchmark's data and verification harness are not publicly released.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ab69166eb290d1e70f471e6c40ad2ace2ebbfc098fc1b206de14ef93048de688"},"motivation":"Coding agents increasingly act as codebase-scale collaborators that can assist with codebase conversion, but this progress has exposed a critical weakness: agents often over-trust their own local validation routines and declare success on artifacts that satisfy surface checks while violating the semantic contracts users actually care about.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29054","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_suichat-cn_1a4b7900","familyId":"bmf_8261cf7b37ba","name":"SuiChat-CN","oneLine":"SuiChat-CN is a Chinese group-chat benchmark for contextual suicide risk assessment, with 13,312 segments from 1,406 users.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27911","pdf":"https://arxiv.org/pdf/2605.27911","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27911"},"evidence":{"snippet":"We introduce SuiChat-CN, a Chinese group-chat benchmark for contextual suicide risk assessment.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27911"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SuiChat-CN is a Chinese group-chat benchmark for contextual suicide risk assessment, with 13,312 segments from 1,406 users.","whyItMatters":"Pioneers risk assessment in group chats but the dataset is restricted for ethical reasons, limiting its public use.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"88488da6ef2109333436f60cf7881eae7dfad1584f6bd4bae1ce2a7d77feba7a"},"motivation":"Suicide is a critical global public health challenge, causing approximately 720,000 deaths each year and calling for timely, effective prevention strategies.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27911","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_storylensbench_8f70dc12","familyId":"bmf_de15c99faa91","name":"STORYLENSBENCH","oneLine":"STORYLENSBENCH benchmarks preference-aligned story rewriting with structured story books and reader profiles, plus reward model and rewriting model.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.28073","pdf":"https://arxiv.org/pdf/2605.28073","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28073"},"evidence":{"snippet":"Motivated by this, we introduce STORYLENSBENCH, a large-scale benchmark for preference-aligned story rewriting, comprising structured story books, multi-dimensional reader preference profiles, and ranked context-aware rewritten stories.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28073"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"STORYLENSBENCH benchmarks preference-aligned story rewriting with structured story books and reader profiles, plus reward model and rewriting model.","whyItMatters":"Focuses on context-aware narrative enrichment, but the benchmark is not publicly released; only the models are proposed.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"77833d44ed605c5d6a0ab1a802b2f1b90b9af11487fc6d42d8d212ea5705ad9c"},"motivation":"Story rewriting aims to adapt existing narratives to diverse reader preferences while preserving plot consistency and narrative coherence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28073","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_spatialbench-long_33cb2dcb","familyId":"bmf_0da3e779f3e3","name":"SpatialBench-Long","oneLine":"SpatialBench-Long evaluates AI agents on long-horizon spatial biology tasks, requiring recovery of biological claims from raw data across 24 evaluations.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.28065","pdf":"https://arxiv.org/pdf/2605.28065","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28065"},"evidence":{"snippet":"We introduce SpatialBench-Long, a benchmark for long-horizon spatial biology in which agents must recover biological claims from raw or near-raw data and calibrated experimental context without prescribed methods.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28065"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"SpatialBench-Long evaluates AI agents on long-horizon spatial biology tasks, requiring recovery of biological claims from raw data across 24 evaluations.","whyItMatters":"Tests agents' ability to synthesize scientific conclusions from complex spatial data, but no public data or code release is mentioned.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8f1d8ec13af1d6b9cd390a0525d15956e4c98483bff966102577795ae15c4ec3"},"motivation":"AI agents are increasingly useful for biological data analysis, but existing benchmarks mostly test broad biological knowledge, executable workflows, or localized analysis steps rather than end-to-end scientific reasoning over spatial measurements.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28065","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_qwen-image-bench_7fd96584","familyId":"bmf_6aa256ab0afc","name":"Qwen-Image-Bench","oneLine":"Qwen-Image-Bench evaluates text-to-image models on five pillars including Real-world Fidelity and Creative Generation, with 1000 prompts and 56 rubric-based facets scored by a trained judge model.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.28091","pdf":"https://arxiv.org/pdf/2605.28091","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28091"},"evidence":{"snippet":"To address the gap, we introduce Qwen-Image-Bench, a creator-centric benchmark co-designed with professional artists and grounded in real-world creation scenarios.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":5,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28091"},"ranking":{"90d":{"score":52,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Qwen-Image-Bench evaluates text-to-image models on five pillars including Real-world Fidelity and Creative Generation, with 1000 prompts and 56 rubric-based facets scored by a trained judge model.","whyItMatters":"It aims to address gaps in existing T2I benchmarks by assessing application-driven capabilities for professional creative workflows, offering fine-grained diagnostics for model comparison and development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ccd3df3cfb163ea85404c59d4466f421e5f52fc8246bf5469ed60dc68adbeefa"},"motivation":"Text-to-Image generation has evolved from basic image synthesis into a frequently used core capability in professional creative workflows, where simple text-image alignment can no longer satisfy users' pressing demands for faithful real-world reconstruction and genuine creative expression.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28091","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_prionner_f3eb1b08","familyId":"bmf_80133507f6df","name":"PrionNER","oneLine":"Named entity recognition dataset from PubMed abstracts on prion disease, with 317 abstracts annotated for 15 coarse and 31 fine-grained entity types, plus evaluation scripts and train/test splits.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2605.28375","pdf":"https://arxiv.org/pdf/2605.28375","project":null,"code":"https://github.com/daotuanan/PrionNER/","data":null,"hfPaper":"https://huggingface.co/papers/2605.28375"},"evidence":{"snippet":"We benchmark supervised BERT baselines, W2NER, and zero-shot extractors on PrionNER.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28375"},"ranking":{"90d":{"score":42,"rank":259,"coverage":0.7,"confidence":"Medium"}},"description":"Named entity recognition dataset from PubMed abstracts on prion disease, with 317 abstracts annotated for 15 coarse and 31 fine-grained entity types, plus evaluation scripts and train/test splits.","whyItMatters":"Fills a gap in biomedical NLP for rare diseases, enabling evaluation of fine-grained and discontinuous entity extraction under low-resource conditions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f9ed7bfe78d23c31f858c49476b9b514b7d6b385da238a5c14cf2dc0005a9c9a"},"motivation":"Prion diseases are rare, rapidly progressive, and fatal neurodegenerative disorders that remain difficult to diagnose, particularly in their early stages because of nonspecific clinical presentations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ACL 25th Workshop on Biomedical Language Processing (BioNLP 2026)","evidence":"29 pages, 5 figures, accepted at ACL 25th Workshop on Biomedical Language Processing (BioNLP 2026)","evidenceUrl":"https://arxiv.org/abs/2605.28375","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ACL 25th Workshop on Biomedical Language Processing (BioNLP 2026)","reviewStatus":"accepted","decisionRaw":"29 pages, 5 figures, accepted at ACL 25th Workshop on Biomedical Language Processing (BioNLP 2026)","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2605.28375","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"29 pages, 5 figures, accepted at ACL 25th Workshop on Biomedical Language Processing (BioNLP 2026)","level":"author-claim"}]}],"publishers":[{"name":"PrionNER team","organizationType":"community","sourceUrl":"https://github.com/daotuanan/PrionNER/","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_portbench_32129f61","familyId":"bmf_1aa870031c3f","name":"PortBench","oneLine":"PortBench evaluates LLM-driven portfolio management via a static QA dataset (6,269 questions across seven task templates) and a dynamic five-stage allocation pipeline, spanning six asset classes over ten years. Scoring includes a dual-layer correlation score and CEPS, with evaluation under three stress regimes and investor profiles.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27887","pdf":"https://arxiv.org/pdf/2605.27887","project":"https://portbench.github.io/","code":"https://github.com/AgenticFinLab/portbench","data":null,"hfPaper":"https://huggingface.co/papers/2605.27887"},"evidence":{"snippet":"We introduce PortBench, a benchmark spanning six heterogeneous asset classes over ten years.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":8,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27887"},"ranking":{"90d":{"score":52,"rank":184,"coverage":0.7,"confidence":"Medium"}},"description":"PortBench evaluates LLM-driven portfolio management via a static QA dataset (6,269 questions across seven task templates) and a dynamic five-stage allocation pipeline, spanning six asset classes over ten years. Scoring includes a dual-layer correlation score and CEPS, with evaluation under three stress regimes and investor profiles.","whyItMatters":"Existing financial benchmarks often ignore cross-asset correlations and the full portfolio management pipeline. PortBench addresses this gap by providing a comprehensive, reusable evaluation for LLM capabilities in realistic portfolio management, enabling comparison across models and informing practical deployment decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b514623ab90b9974c54f11299d7b16c95e80565ce3ccbe1091959579bfd21376"},"motivation":"Large language models (LLMs) have shown strong performance across diverse financial tasks, yet portfolio management (PM) remains poorly benchmarked.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27887","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AgenticFinLab","organizationType":"academic-lab","sourceUrl":"https://github.com/AgenticFinLab/portbench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_pointq-bench_51211dca","familyId":"bmf_6e95a65fb614","name":"PointQ-Bench","oneLine":"PointQ-Bench is a benchmark for point cloud quality assessment, extending from scalar scoring to comprehensive quality understanding, with 3,083 point clouds and tasks like anomaly sensing, defect diagnosis, usability grading, and open-ended quality reporting.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.28241","pdf":"https://arxiv.org/pdf/2605.28241","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28241"},"evidence":{"snippet":"We introduce PointQ-Bench, a benchmark designed to extend PCQA from scalar scoring toward comprehensive quality understanding.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28241"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PointQ-Bench is a benchmark for point cloud quality assessment, extending from scalar scoring to comprehensive quality understanding, with 3,083 point clouds and tasks like anomaly sensing, defect diagnosis, usability grading, and open-ended quality reporting.","whyItMatters":"Current PCQA benchmarks focus on scalar prediction, leaving gaps in diagnostic and interpretable quality assessment. PointQ-Bench addresses this by evaluating models on multi-faceted quality understanding tasks, which is crucial for practical inspection scenarios where identifying defects and assessing usability is necessary.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"7806450d88448de6f425cb752c2577b82256e1ca77d0b2d57f39cd1d2d5bf2be"},"motivation":"Point cloud quality plays a critical role in 3D acquisition, reconstruction, rendering, and perception, yet existing point cloud quality assessment (PCQA) research remains largely centered on scalar score prediction.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28241","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_poinav-bench_7e6a4ee3","familyId":"bmf_aca8b20281af","name":"POINav-Bench","oneLine":"POINav-Bench evaluates vision-language navigation agents in real-world POI-goal navigation across 11 reconstructed commercial areas covering 126,398 m² with 163 POIs, using traversability-aware annotations and reference trajectories for closed-loop evaluation.","area":"Robotics & Embodied AI","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.28237","pdf":"https://arxiv.org/pdf/2605.28237","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28237"},"evidence":{"snippet":"To bridge this gap, we present POINav-Bench, the first benchmark designed for closed-loop evaluation of real-world POI-goal navigation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28237"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"POINav-Bench evaluates vision-language navigation agents in real-world POI-goal navigation across 11 reconstructed commercial areas covering 126,398 m² with 163 POIs, using traversability-aware annotations and reference trajectories for closed-loop evaluation.","whyItMatters":"Existing VLN benchmarks for POI-goal navigation suffer from coarse granularity or sim-to-real gaps. POINav-Bench provides high-fidelity real-world environments, enabling evaluation of final-meters navigation capabilities that are critical for practical deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"a765471fff8234d74b41ab3f4bab81b320d84dfbdde8b5f4da1a57fbabb8a576"},"motivation":"Real-world navigation is fundamentally driven by Points of Interest (POIs), yet reaching a precise POI remains a critical \"final-meters\" challenge.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28237","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"POINav Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2605.28237","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"general"},{"id":"bm_peft-arena_b7338aaa","familyId":"bmf_f5e847eb890b","name":"PEFT-Arena","oneLine":"PEFT-Arena benchmark jointly evaluates target-domain performance and retention of pretrained capabilities for parameter-efficient finetuning methods. It covers mathematical and medical reasoning as target domains and measures general capability retention on benchmarks including BBH, IFEval, and NQ with SFT and RLVR training settings.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.28819","pdf":"https://arxiv.org/pdf/2605.28819","project":"https://spherelab.ai/PEFT-Arena/","code":"https://github.com/Sphere-AI-Lab/PEFT-Arena","data":null,"hfPaper":"https://huggingface.co/papers/2605.28819"},"evidence":{"snippet":"We introduce PEFT-Arena, a benchmark that jointly measures downstream performance and general capability retention.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":9,"hfDailySubmittedAt":"2026-05-28T00:00:00.000Z","githubStars":28,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28819"},"ranking":{"90d":{"score":69,"rank":80,"coverage":0.7,"confidence":"Medium"}},"description":"PEFT-Arena benchmark jointly evaluates target-domain performance and retention of pretrained capabilities for parameter-efficient finetuning methods. It covers mathematical and medical reasoning as target domains and measures general capability retention on benchmarks including BBH, IFEval, and NQ with SFT and RLVR training settings.","whyItMatters":"Existing PEFT evaluations focus mainly on downstream accuracy, overlooking retention of pretrained abilities. This benchmark provides a stability-plasticity perspective, enabling selection of fine-tuning methods that balance task adaptation and forgetting resistance, offering a more complete assessment for practical deployment decisions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"deabbebd645b7ebe77205d30ddd4a8cce565c608114c85c53cd71e0d4157c0bd"},"motivation":"Parameter-efficient finetuning (PEFT) has become the standard approach for adapting large language models, yet evaluations largely emphasize downstream accuracy while overlooking the retention of pretrained capabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28819","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Sphere AI Lab","organizationType":"academic-lab","sourceUrl":"https://spherelab.ai/PEFT-Arena/","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_or-space_6e200388","familyId":"bmf_3a4d8160cbad","name":"OR-Space","oneLine":"A full-lifecycle workspace benchmark for industrial optimization agents, evaluating model construction, revision, and grounded explanation across three task modes (Build, Revise, Explain) using executable multi-file workspaces with task-specific evaluators.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.28158","pdf":"https://arxiv.org/pdf/2605.28158","project":null,"code":"https://github.com/0xzhouchenyu/OR-Space","data":null,"hfPaper":"https://huggingface.co/papers/2605.28158"},"evidence":{"snippet":"We introduce OR-Space, a full-lifecycle workspace benchmark for evaluating industrial optimization agents across model construction, model revision, and grounded explanation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":6,"hfDailySubmittedAt":"2026-05-28T00:00:00.000Z","githubStars":12,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28158"},"ranking":{"90d":{"score":62,"rank":118,"coverage":0.7,"confidence":"Medium"}},"description":"A full-lifecycle workspace benchmark for industrial optimization agents, evaluating model construction, revision, and grounded explanation across three task modes (Build, Revise, Explain) using executable multi-file workspaces with task-specific evaluators.","whyItMatters":"Fills the gap in benchmarking LLM agents for real industrial OR workflows, where persistent multi-artifact workspaces and multi-stage lifecycles are central, offering a more realistic evaluation of practical readiness beyond single-shot formulation tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"2275bbeddc868adc4d82c8d69ac82a5139372958660412e16d8e3ccf0ac8f78d"},"motivation":"Large language model (LLM) agents are increasingly used to assist with operations research (OR) modeling, yet existing OR-oriented benchmarks often reduce evaluation to one-shot translation from a self-contained problem statement into a mathematical formulation or solver program.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28158","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"OR-Space team","organizationType":"academic-lab","sourceUrl":"https://github.com/0xzhouchenyu/OR-Space","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_muse_8c1c770b","familyId":"bmf_4016c3db3bc3","name":"MUSE","oneLine":"MUSE evaluates text-to-CAD generation of complex B-Rep assemblies via design specifications. It scores models on code validity, geometric correctness, and design-intent alignment using rubrics covering functionality, manufacturability, and assemblability. A VLM judge with human validation is used for scalable scoring.","area":"Science & Engineering","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Geometric reasoning"],"topics":["CAD"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.28579","pdf":"https://arxiv.org/pdf/2605.28579","project":"https://dong7313.github.io/muse-benchmark/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28579"},"evidence":{"snippet":"To address this gap, we introduce MUSE, a Text-to-CAD benchmark focused on complex, editable boundary representation (B-Rep) assemblies.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28579"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"MUSE evaluates text-to-CAD generation of complex B-Rep assemblies via design specifications. It scores models on code validity, geometric correctness, and design-intent alignment using rubrics covering functionality, manufacturability, and assemblability. A VLM judge with human validation is used for scalable scoring.","whyItMatters":"Existing CAD benchmarks focus on single-part geometric similarity, missing industrial requirements. MUSE provides a structured evaluation that measures practical design quality, enabling progress toward engineering-ready CAD generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0397b51cc33fa03c4e4e04f9ab59cf3a3d9d9e5a19cbcf53a731bb6a569cb245"},"motivation":"Large language models (LLMs) have recently advanced text-driven 3D generation, yet Text-to-CAD remains far from supporting industrial product design.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28579","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MUSE Benchmark Team","organizationType":"academic-lab","sourceUrl":"https://dong7313.github.io/muse-benchmark/","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_mtavg-bench_0372b3a9","familyId":"bmf_ac59d735d593","name":"MTAVG-Bench","oneLine":"MTAVG-Bench 2.0 evaluates omni large language models on diagnosing high-level cinematic failures in multi-talker audio-video generation, with over 10,000 QA instances covering acting, narrative, atmosphere, and audio-visual language.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.28035","pdf":"https://arxiv.org/pdf/2605.28035","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28035"},"evidence":{"snippet":"To fill this gap, we introduce MTAVG-Bench 2.0, a benchmark for diagnosing failure modes of cinematic expressiveness in multi-talker audio-video generation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28035"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MTAVG-Bench 2.0 evaluates omni large language models on diagnosing high-level cinematic failures in multi-talker audio-video generation, with over 10,000 QA instances covering acting, narrative, atmosphere, and audio-visual language.","whyItMatters":"Standard metrics like lip-sync do not capture cinematic expressiveness; this benchmark targets a gap in evaluating higher-level audio-visual quality in scene-level generation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0dbaa6e642f474af1f621756a2d6f20f876f9c4ea1b54424b218d5de7b88e490"},"motivation":"In recent years, Multi-Talker Audio-Video Generation (MTAVG) models have shown promising performance on fundamental metrics such as lip-sync and audio-visual alignment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28035","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mov-bench_db8a731e","familyId":"bmf_5a9bcf5cba7a","name":"MOV-Bench","oneLine":"MOV-Bench contains 519 questions requiring multi-hop reasoning over temporally dispersed audio-visual evidence, evaluating omni-modal LLMs on cross-modal reasoning tasks.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.28192","pdf":"https://arxiv.org/pdf/2605.28192","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28192"},"evidence":{"snippet":"In this work, we introduce MOV-Bench, a benchmark containing 519 carefully curated questions that require multi-hop reasoning over temporally dispersed audio-visual evidence.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28192"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MOV-Bench contains 519 questions requiring multi-hop reasoning over temporally dispersed audio-visual evidence, evaluating omni-modal LLMs on cross-modal reasoning tasks.","whyItMatters":"Existing benchmarks offer limited investigation of multi-hop audio-visual reasoning; MOV-Bench provides a focused evaluation set for this capability.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c097da2aa1043f63dd1e2aeae4f8aab528bb6dc2480d3cb2ba68281e33fb4f62"},"motivation":"Multi-hop audio-visual reasoning remains challenging for Omni-LLMs, as relevant evidence is often sparse, temporally dispersed, and distributed across both audio and visual streams.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28192","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_meniomni_7f0a7094","familyId":"bmf_7f40c7d5cfda","name":"MeniOmni","oneLine":"MeniOmni is a multimodal benchmark for meniscus injury assessment with 746 MRI studies, supporting Stoller severity grading and diagnostic report generation, with risk-aware ordinal evaluation and semantic consistency metric.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.28161","pdf":"https://arxiv.org/pdf/2605.28161","project":null,"code":"https://github.com/ShuruiXu/MeniOmni","data":null,"hfPaper":"https://huggingface.co/papers/2605.28161"},"evidence":{"snippet":"We introduce MeniOmni, a structured multimodal benchmark for meniscus injury assessment, consisting of 746 multi-center MRI studies with tri-planar volumetric inputs, Clinical Priors, and expert-annotated clinical text.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":0,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28161"},"ranking":{"90d":{"score":29,"rank":379,"coverage":0.55,"confidence":"Low"}},"description":"MeniOmni is a multimodal benchmark for meniscus injury assessment with 746 MRI studies, supporting Stoller severity grading and diagnostic report generation, with risk-aware ordinal evaluation and semantic consistency metric.","whyItMatters":"Addresses a gap in knee MRI benchmarks by integrating volumetric MRI data with clinical context for holistic clinical reasoning, aiming to improve safety in meniscus injury diagnosis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"86f783e39c54113133330e0e018d95c1ac29899bc47185a0cd2a433602f79f19"},"motivation":"Clinical diagnosis of meniscus injuries requires radiologists to integrate volumetric MRI evidence with patient context (e.g., sex, age, BMI) and to produce structured diagnostic reports.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"IEEE International Conference on Multimedia and Expo (ICME) 2026 (Oral Presentation)","evidence":"Accepted by IEEE International Conference on Multimedia and Expo (ICME) 2026 (Oral Presentation)","evidenceUrl":"https://arxiv.org/abs/2605.28161","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"IEEE International Conference on Multimedia and Expo (ICME) 2026 (Oral Presentation)","reviewStatus":"accepted","decisionRaw":"Accepted by IEEE International Conference on Multimedia and Expo (ICME) 2026 (Oral Presentation)","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2605.28161","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by IEEE International Conference on Multimedia and Expo (ICME) 2026 (Oral Presentation)","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_ksafe-mm_8d1d1de0","familyId":"bmf_f2d785a9edd4","name":"KSAFE-MM","oneLine":"KSAFE-MM evaluates multimodal LLM safety in Korean contexts, with 12 models tested on general and culture-specific safety risks, including jailbreak-style textual queries paired with local visual cues.","area":"Safety & Trustworthiness","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["Multimodal","Safety"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.28013","pdf":"https://arxiv.org/pdf/2605.28013","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28013"},"evidence":{"snippet":"This paper introduces KSAFE-MM, a benchmark for Korean multimodal safety evaluation that covers both general safety risks and culture-specific vulnerabilities.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28013"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"KSAFE-MM evaluates multimodal LLM safety in Korean contexts, with 12 models tested on general and culture-specific safety risks, including jailbreak-style textual queries paired with local visual cues.","whyItMatters":"Existing safety benchmarks are English-centric and ignore local cultural risks; KSAFE-MM provides a general-to-local pipeline for culturally grounded safety evaluation, revealing trade-offs between safety and over-refusal.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"66ceda06edc2c977604ffe5ad1044258a296b89d6dfdb939d360444ec8a71f6a"},"motivation":"Multimodal Large Language Models (MLLMs) exacerbate safety risks by introducing vulnerabilities across multiple modalities, such as language and vision.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28013","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"specific"},{"id":"bm_ifmtbench_b83f8c4f","familyId":"bmf_c2644d034b9f","name":"IFMTBench","oneLine":"Evaluates multilingual translation systems on instruction following across seven languages, six constraint types, and compositional multi-constraint requests.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2605.28218","pdf":"https://arxiv.org/pdf/2605.28218","project":null,"code":"https://github.com/Tencent-Hunyuan/Hy-MT2/tree/main/IFMTBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.28218"},"evidence":{"snippet":"We introduce \\bench, a benchmark for multilingual translation instruction following covering seven languages, with 4,506 single-constraint and 2,838 multi-constraint items spanning six constraint dimensions and five compositional patterns with instructions issued in all seven languages.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":554,"githubScope":"hosting_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28218"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates multilingual translation systems on instruction following across seven languages, six constraint types, and compositional multi-constraint requests.","whyItMatters":"Shows whether translation systems can follow terminology, style, length, audience, and other workflow constraints while preserving translation quality.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8ffde504a4f6f0125b70268f3de10201211ac5b384be2b71235cdc817cda7118"},"motivation":"Modern translation workflows demand more than semantic equivalence.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28218","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Tencent Hunyuan","organizationType":"company-research-lab","role":"benchmark-publisher","sourceUrl":"https://github.com/Tencent-Hunyuan/Hy-MT2/tree/main/IFMTBench"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_harness-bench_b6a94bb9","familyId":"bmf_5d1a38309c97","name":"Harness-Bench","oneLine":"Harness-Bench evaluates configuration-level harness effects in agent workflows with 106 sandboxed tasks, measuring completion, process quality, efficiency, and failure behavior across model-harness pairings.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27922","pdf":"https://arxiv.org/pdf/2605.27922","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27922"},"evidence":{"snippet":"We introduce Harness-Bench, a diagnostic benchmark for evaluating configuration-level harness effects in realistic agent workflows.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27922"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Harness-Bench evaluates configuration-level harness effects in agent workflows with 106 sandboxed tasks, measuring completion, process quality, efficiency, and failure behavior across model-harness pairings.","whyItMatters":"Addresses a gap in agent evaluation by isolating harness configuration effects, showing that agent capability is configuration-level rather than model-only, and identifying execution-alignment failures.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9ebfc8b12156538dfa2d5e20681e9fc36cd36c0044c08f436d2d6be2156ac499"},"motivation":"LLM agents are increasingly deployed as executable systems that use tools, modify workspaces, and produce concrete artifacts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27922","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_hardmtbench_3eaf8a99","familyId":"bmf_91528a6e04ed","name":"HardMTBench","oneLine":"HardMTBench is a difficulty-aware diagnostic benchmark for Chinese-English domain translation, covering 12 domains with 20,000 directional test items and annotated hardness scores based on domain knowledge, translation difficulty, and terminology load.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.28315","pdf":"https://arxiv.org/pdf/2605.28315","project":null,"code":"https://github.com/jasonNLP/HardMTBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.28315"},"evidence":{"snippet":"We introduce HardMTBench, a difficulty-aware diagnostic benchmark for bidirectional Chinese-English domain translation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28315"},"ranking":{"90d":{"score":44,"rank":248,"coverage":0.55,"confidence":"Low"}},"description":"HardMTBench is a difficulty-aware diagnostic benchmark for Chinese-English domain translation, covering 12 domains with 20,000 directional test items and annotated hardness scores based on domain knowledge, translation difficulty, and terminology load.","whyItMatters":"Addresses the saturation of general MT benchmarks on Chinese-English by widening score separation, exposing domain-specific weaknesses in knowledge-intensive areas that quality-only metrics miss.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b9f7f5ff378782dbc14a56d78cc76f8200856ce363e51fbafcf0000647111757"},"motivation":"General-purpose machine translation benchmarks such as FLORES-200 have reached a saturation regime on Chinese-English pairs, where modern large language models cluster within a narrow band of high scores.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28315","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_geo-bench_c7e53a2d","familyId":"bmf_cbe17d8c1c26","name":"GEO-Bench","oneLine":"GEO-Bench evaluates ranking manipulation attacks in generative engine optimization under a unified protocol, measuring effectiveness and stealth across five datasets against a fixed ranker, with open-source attack implementations.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Robot manipulation"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.29107","pdf":"https://arxiv.org/pdf/2605.29107","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29107"},"evidence":{"snippet":"We present GEO-Bench, a benchmark that evaluates GEO ranking-manipulation attacks under one protocol.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29107"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"GEO-Bench evaluates ranking manipulation attacks in generative engine optimization under a unified protocol, measuring effectiveness and stealth across five datasets against a fixed ranker, with open-source attack implementations.","whyItMatters":"Addresses the lack of standardized comparison in GEO attack research, enabling direct comparison across attack paradigms and supporting detection development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fbc9be61c8075dbaf02cc15489fa211cf86bd400e745dfc5be69d6607aff53de"},"motivation":"Large language models (LLMs) increasingly rank products, documents, and recommendations for user queries, which makes manipulating these rankings a growing concern for fairness and information integrity.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29107","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_forcebench_de197b73","familyId":"bmf_3be86d47de12","name":"FORCEBENCH","oneLine":"FORCEBENCH is a contrastive stress test for evidence-force calibration in cited RAG. It pairs fixed cited passages with evidence-calibrated claims and force-raised variants across five axes: relation, modality, scope, temporal validity, and numeric specificity. Evaluation is a fixed, locality-filtered set of 198 pairs.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.28044","pdf":"https://arxiv.org/pdf/2605.28044","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28044"},"evidence":{"snippet":"We introduce FORCEBENCH, a contrastive stress test for evidence-force calibration.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28044"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FORCEBENCH is a contrastive stress test for evidence-force calibration in cited RAG. It pairs fixed cited passages with evidence-calibrated claims and force-raised variants across five axes: relation, modality, scope, temporal validity, and numeric specificity. Evaluation is a fixed, locality-filtered set of 198 pairs.","whyItMatters":"Current cited RAG evaluation often treats topical relevance as sufficient grounding, missing cases where a relevant source under-warrants an over-strong claim. FORCEBENCH exposes this citation laundering failure and measures evaluator calibration via monotonicity violation rate and force sensitivity.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ee77919ebc271f871f95c08b31ee3200e7432ef59af1bf20dd9ed5cf8c43053e"},"motivation":"Cited RAG evaluation often treats visible sources as a grounding signal, but a real, topically relevant citation can still under-warrant the attached wording.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28044","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_finboardbench_0e7661a6","familyId":"bmf_f6dd71cc3fce","name":"FinBoardBench","oneLine":"FinBoardBench is an evaluation suite based on three financial board games: Cashflow, Acquire, and Monopoly. It assesses financial skills including personal cash flow management, corporate investment and acquisition forecasting, and competitive trade negotiations with asset auctions, using game simulations as the environment.","area":"Language & Knowledge","applicationDomains":["Finance & Economics"],"primaryDomain":"Finance & Economics","industrySectors":["Financial Services"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27896","pdf":"https://arxiv.org/pdf/2605.27896","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27896"},"evidence":{"snippet":"To bridge this gap, we present FinBoardBench, an evaluation suite based on three classic financial board games: Cashflow, Acquire, and Monopoly.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27896"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"FinBoardBench is an evaluation suite based on three financial board games: Cashflow, Acquire, and Monopoly. It assesses financial skills including personal cash flow management, corporate investment and acquisition forecasting, and competitive trade negotiations with asset auctions, using game simulations as the environment.","whyItMatters":"Existing static financial benchmarks do not capture dynamic decision-making in complex environments. FinBoardBench provides a repeatable protocol to measure whether LLMs can translate static reasoning into successful dynamic action, addressing a gap in financial AI evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"de9916c6d32cee47c6a8c512ac3d61c05c365793cc2b93a2a054141276c34b51"},"motivation":"Recently, large language models (LLMs) have achieved superior performance in static financial reasoning and simple dynamic trading tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27896","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_embodied3dbench_d183e439","familyId":"bmf_1a1023e37b98","name":"Embodied3DBench","oneLine":"Embodied3DBench is a robot-centric benchmark for low-level spatial intelligence in embodied 3D environments. It includes 6 task categories: Grounding, Spatial Relation Prediction, Multi-view Correspondence, Affordance Prediction, Grasp Point Prediction, and Trajectory Prediction, with 21k QA pairs across 12 subcategories.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":["Geometric reasoning"],"topics":["Robotics"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.29074","pdf":"https://arxiv.org/pdf/2605.29074","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.29074"},"evidence":{"snippet":"We introduce Embodied3DBench, a robot-centric benchmark targeting low-level spatial intelligence in embodied 3D environments.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.29074"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Embodied3DBench is a robot-centric benchmark for low-level spatial intelligence in embodied 3D environments. It includes 6 task categories: Grounding, Spatial Relation Prediction, Multi-view Correspondence, Affordance Prediction, Grasp Point Prediction, and Trajectory Prediction, with 21k QA pairs across 12 subcategories.","whyItMatters":"Current VLMs show strong high-level spatial reasoning but lack interaction-oriented perception. Embodied3DBench reveals this gap and provides a scalable training dataset for improvement, enabling systematic evaluation and advancement of interaction-aware multimodal systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"49d23f33ffa8b41b6907b442b42771c8d6a65a7cab2d758b5f8fa1190032c601"},"motivation":"Are current Vision Language Models (VLMs) ready to comprehend and reason about complex embodied interactions in 3D environments?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.29074","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_egobench_d0f8580c","familyId":"bmf_31eeb47a8e16","name":"EgoBench","oneLine":"EgoBench is an interactive multimodal benchmark for tool-using agents. It comprises 1,045 egocentric-video-grounded tasks across four daily scenarios, with a user-agent-tool interactive environment. It assesses multimodal perception, tool invocation with multi-hop reasoning, and dynamic user interaction.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27820","pdf":"https://arxiv.org/pdf/2605.27820","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27820"},"evidence":{"snippet":"To bridge this gap, we introduce EgoBench, the first interactive multimodal benchmark for tool-using agents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27820"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"EgoBench is an interactive multimodal benchmark for tool-using agents. It comprises 1,045 egocentric-video-grounded tasks across four daily scenarios, with a user-agent-tool interactive environment. It assesses multimodal perception, tool invocation with multi-hop reasoning, and dynamic user interaction.","whyItMatters":"AI agents in open environments need joint multimodal and tool-use capabilities. EgoBench provides a standardized interactive environment and deterministic joint validation to objectively measure these skills, addressing a lack of comparable benchmarks for dynamic tool-using agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"51ad36a0df77af5d2ff7bfe561f6ac08591c213b89d6f85fee1d5df9e6633fd5"},"motivation":"As AI agents increasingly operate in open, real-world environments, they require a deep synergy of multimodal perception, tool invocation with multi-hop reasoning, and dynamic interaction with users.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27820","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_dr-cik_017f835a","familyId":"bmf_6f7e7a58db67","name":"Dr-CiK","oneLine":"Dr-CiK is a benchmark for evaluating whether agents can retrieve forecasting-relevant supporting context from a document corpus, filter distractors, distill evidence, and generate forecasts. It provides context ablations and evaluates deep research and forecasting methods.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27904","pdf":"https://arxiv.org/pdf/2605.27904","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27904"},"evidence":{"snippet":"Therefore, we introduce Dr-CiK, a benchmark for evaluating whether agents can retrieve forecasting-relevant supporting context from a document corpus, filter out distractors, distill the retrieved context into forecast-useful evidence, and generate forecasts supported by that evidence.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27904"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Dr-CiK is a benchmark for evaluating whether agents can retrieve forecasting-relevant supporting context from a document corpus, filter distractors, distill evidence, and generate forecasts. It provides context ablations and evaluates deep research and forecasting methods.","whyItMatters":"Real-world forecasting requires active discovery of external context from noisy sources. Dr-CiK assesses the entire pipeline of context retrieval and use, revealing that most agents recover little evidence and are misled by distractors, guiding development of foresight-driven agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c646db2a54322b4c8e26f2be5bc60cf8e4b7041375f56757268633a575e86d29"},"motivation":"Time series forecasting in real-world settings often depends not only on historical observations, but also on external context that must be actively discovered from noisy, heterogeneous information sources.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27904","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_disasterbench_d2342204","familyId":"bmf_70b465e708e7","name":"DisasterBench","oneLine":"DisasterBench is a benchmark for evaluating structured multi-agent planning over disaster-response tools. It includes 233 expert-verified tasks, 26 agents, 81 typed compatibility edges, and 5 planning paradigms. It uses First-Point-of-Failure (FPoF) for step-level failure attribution.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2605.27957","pdf":"https://arxiv.org/pdf/2605.27957","project":null,"code":"https://github.com/TamuChen18/DisasterBench_Open","data":null,"hfPaper":"https://huggingface.co/papers/2605.27957"},"evidence":{"snippet":"We introduce DisasterBench, a benchmark for evaluating structured multi-agent planning over semantically similar but operationally distinct disaster-response tools.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27957"},"ranking":{"90d":{"score":44,"rank":249,"coverage":0.55,"confidence":"Low"}},"description":"DisasterBench is a benchmark for evaluating structured multi-agent planning over disaster-response tools. It includes 233 expert-verified tasks, 26 agents, 81 typed compatibility edges, and 5 planning paradigms. It uses First-Point-of-Failure (FPoF) for step-level failure attribution.","whyItMatters":"Disaster response requires orchestrating heterogeneous AI tools into executable workflows. DisasterBench tests grounding under typed interface constraints, highlighting gaps between semantic reasoning and execution consistency, and providing diagnostics for failure attribution.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f07ac4e1004cd46c3a46982e0796b8416bcf07bc68373ac843ced15fc3ba003d"},"motivation":"Disasters cause severe societal impacts, demanding rapid coordination of heterogeneous AI tools, from satellite analysis to flood prediction and damage assessment, into coherent multi-step workflows.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27957","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DisasterBench Team","organizationType":"academic-lab","sourceUrl":"https://github.com/TamuChen18/DisasterBench_Open","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_childeval_9775399e","familyId":"bmf_61cce4f98ffa","name":"ChildEval","oneLine":"ChildEval is a benchmark for evaluating LLMs' ability to infer and follow child-centered preferences in long-context conversations. It contains 29K synthesized persona profiles of children aged 3-6, with explicit and implicit preference expressions across five top-level and fourteen sub-level categories.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Long Context"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27805","pdf":"https://arxiv.org/pdf/2605.27805","project":null,"code":"https://github.com/ziyanluo/ChildEval","data":null,"hfPaper":"https://huggingface.co/papers/2605.27805"},"evidence":{"snippet":"To address this gap, we introduce ChildEval, a benchmark for evaluating LLMs' ability to infer and follow child-centered preferences in long-context conversations.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":1,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27805"},"ranking":{"90d":{"score":39,"rank":299,"coverage":0.55,"confidence":"Low"}},"description":"ChildEval is a benchmark for evaluating LLMs' ability to infer and follow child-centered preferences in long-context conversations. It contains 29K synthesized persona profiles of children aged 3-6, with explicit and implicit preference expressions across five top-level and fourteen sub-level categories.","whyItMatters":"Personalization for children is under-explored relative to adults. ChildEval provides a protocol to test whether LLMs can infer and follow child-specific preferences, addressing a gap in personalized conversational AI evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"17dc3cabe5cc52157285d91e37e47bd5f0faaa2b978a936491bf9f3f1c02adc8"},"motivation":"While LLMs enable personalized chatbots, their effectiveness in child-centered personalization remains unclear, as systematic evaluation of child-specific preferences is still lacking.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27805","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ChildEval Team","organizationType":"academic-lab","sourceUrl":"https://github.com/ziyanluo/ChildEval","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning","Long Context & Memory"],"domainScope":"general"},{"id":"bm_asynctool_407c214b","familyId":"bmf_ec6315a36e48","name":"AsyncTool","oneLine":"AsyncTool is a benchmark for evaluating asynchronous function calling in multi-task tool-use environments. It presents multiple tasks with simulated tool response latency, assessing step-level tool-call correctness, sub-task completion, and task-level end-to-end success, along with efficiency-oriented metrics.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-27","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27995","pdf":"https://arxiv.org/pdf/2605.27995","project":null,"code":"https://github.com/StoKou/repo-asynctool","data":null,"hfPaper":"https://huggingface.co/papers/2605.27995"},"evidence":{"snippet":"To evaluate it, we propose AsyncTool, a benchmark for assessing LLM-based agents in interactive multi-task tool-use environments with delayed tool feedback.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":16,"hfDailySubmittedAt":"2026-05-29T00:00:00.000Z","githubStars":38,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27995"},"ranking":{"90d":{"score":73,"rank":60,"coverage":0.7,"confidence":"Medium"}},"description":"AsyncTool is a benchmark for evaluating asynchronous function calling in multi-task tool-use environments. It presents multiple tasks with simulated tool response latency, assessing step-level tool-call correctness, sub-task completion, and task-level end-to-end success, along with efficiency-oriented metrics.","whyItMatters":"Real-world tool use involves delays, but existing benchmarks assume immediate responses. AsyncTool measures whether agents can coordinate multiple tasks and use idle time efficiently, identifying key failure modes for temporal reasoning and task coordination.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"95896a723774ae6cbdf64037db9c11076b9462fd539755d0e5c13089ecf1e6f0"},"motivation":"Large language model (LLM)-based agents have shown strong capabilities in using external tools to solve complex tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27995","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"AsyncTool Team","organizationType":"academic-lab","sourceUrl":"https://github.com/StoKou/repo-asynctool","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_vitabench_d7ecb50f","familyId":"bmf_2e1c2e7b149d","name":"VitaBench","oneLine":"VitaBench 2.0 evaluates personalized and proactive agent behavior in long-term, multi-session user interactions across food delivery, in-store consumption, and online travel domains. Tasks are per-user sequences of subtasks requiring agents to infer, utilize, and update user preferences from fragmented interaction history, with an extensible memory interface for controlled comparison across memory architectures.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27141","pdf":"https://arxiv.org/pdf/2605.27141","project":null,"code":"https://github.com/meituan-longcat/vitabench-2.0","data":null,"hfPaper":"https://huggingface.co/papers/2605.27141"},"evidence":{"snippet":"To address this gap, we introduce VitaBench 2.0, a benchmark for evaluating personalized and proactive agent behavior in long-term user interactions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":20,"hfDailySubmittedAt":"2026-05-27T00:00:00.000Z","githubStars":62,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27141"},"ranking":{"90d":{"score":78,"rank":30,"coverage":0.7,"confidence":"Medium"}},"description":"VitaBench 2.0 evaluates personalized and proactive agent behavior in long-term, multi-session user interactions across food delivery, in-store consumption, and online travel domains. Tasks are per-user sequences of subtasks requiring agents to infer, utilize, and update user preferences from fragmented interaction history, with an extensible memory interface for controlled comparison across memory architectures.","whyItMatters":"Existing agent benchmarks focus on reasoning and tool use, overlooking the challenges of inferring and leveraging user preferences over time. This benchmark isolates personalization and proactivity in long-horizon tasks, providing a measurement of practical readiness for life-serving applications where models must act on implicit and evolving user needs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9f68e150f3c43b78ef0ca220cd8ebd80f69ec08e04ed58d8987ee39ec55825b5"},"motivation":"Large language models (LLMs) have evolved into interactive agents that collaborate with users in real-world tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27141","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Meituan","organizationType":"company-research-lab","sourceUrl":"https://github.com/meituan-longcat/vitabench-2.0","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_spatialbench_9dae04ac","familyId":"bmf_0f6a40719bb8","name":"SpatialBench","oneLine":"SpatialBench is a deterministic, density-aware benchmark for spatial foundation models, spanning 19 datasets, 546 scenes, and five spatial domains. It evaluates 41 models across six paradigms on five task suites—depth, camera pose, trajectory, point-cloud reconstruction, and long-sequence streaming—under four input density settings with precomputed and pinned test frames.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27367","pdf":"https://arxiv.org/pdf/2605.27367","project":"https://ropedia.github.io/SpatialBench/","code":"https://github.com/Ropedia/SpatialBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.27367"},"evidence":{"snippet":"To address this gap, we present SpatialBench, a cross-paradigm, domain-diverse benchmark for spatial foundation models with deterministic sampling.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":73,"hfDailySubmittedAt":"2026-05-27T00:00:00.000Z","githubStars":122,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27367"},"ranking":{"90d":{"score":82,"rank":14,"coverage":0.7,"confidence":"Medium"}},"description":"SpatialBench is a deterministic, density-aware benchmark for spatial foundation models, spanning 19 datasets, 546 scenes, and five spatial domains. It evaluates 41 models across six paradigms on five task suites—depth, camera pose, trajectory, point-cloud reconstruction, and long-sequence streaming—under four input density settings with precomputed and pinned test frames.","whyItMatters":"Spatial foundation models are typically evaluated only on domains they were designed for, making cross-domain generalization difficult to assess. SpatialBench provides a controlled protocol with fixed sampling and multiple density settings, enabling a holistic comparison of generalization across viewpoints, scene domains, and hardware constraints.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"505b4cad6d6f880c864a40b68e5d3d888acd56f5bbabfc8c67e7f4c6e3fe42c8"},"motivation":"While spatial foundation models have demonstrated impressive performance on standard datasets, a critical question remains: are they truly all-round players capable of generalizing robustly across diverse downstream tasks, arbitrary viewpoints, shifting scene domains, varying input densities, and specific hardware constraints?","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27367","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Ropedia","organizationType":"academic-lab","sourceUrl":"https://github.com/Ropedia/SpatialBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_sec-bench_29fdf2d8","familyId":"bmf_830a56b48245","name":"SEC-bench","oneLine":"SEC-bench Pro measures long-horizon vulnerability discovery in software security. It includes 344 verified vulnerabilities across V8, SpiderMonkey, and Linux kernel, each paired with instructions for reproducing a working proof-of-concept. Grading uses an LLM-based judge to classify generated PoCs against vulnerable, fixed, and latest images.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26548","pdf":"https://arxiv.org/pdf/2605.26548","project":null,"code":"https://github.com/SEC-bench/SEC-bench-Pro","data":null,"hfPaper":"https://huggingface.co/papers/2605.26548"},"evidence":{"snippet":"We present SEC-bench Pro, a benchmark that measures how well frontier models hunt real vulnerabilities by reproducing working PoC inputs from disclosed reports, where each task pairs a concrete bug with the instructions for triggering it.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":46,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26548"},"ranking":{"90d":{"score":65,"rank":99,"coverage":0.7,"confidence":"Medium"}},"description":"SEC-bench Pro measures long-horizon vulnerability discovery in software security. It includes 344 verified vulnerabilities across V8, SpiderMonkey, and Linux kernel, each paired with instructions for reproducing a working proof-of-concept. Grading uses an LLM-based judge to classify generated PoCs against vulnerable, fixed, and latest images.","whyItMatters":"Finding real vulnerabilities requires reasoning across an entire codebase to produce a working PoC, a challenging task that is understudied. SEC-bench Pro provides a reproducible environment for evaluating long-horizon security capabilities, helping identify where models succeed and fail in realistic vulnerability hunting.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"4a67b69f1e510b57e99bbf96d2c727261215a3f297a7c61b91228934e693307a"},"motivation":"Finding a real vulnerability in complicated systems is a challenging, long-horizon task that demands reasoning across an entire codebase to produce a working proof-of-concept (PoC).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26548","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_receipt-replay-ood_d32b05db","familyId":"bmf_172d0a1e1e4b","name":"Receipt Replay OOD","oneLine":"Receipt Replay OOD is a small out-of-domain benchmark for screen replay detection. It uses receipts, which share planar geometry, curved corners, wear-and-tear artifacts, and text patterns with identity documents, to evaluate document replay detection models under cross-domain conditions without personally identifiable information constraints.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26855","pdf":"https://arxiv.org/pdf/2605.26855","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.26855"},"evidence":{"snippet":"In this work, we introduce Receipt Replay OOD, a small out-of-domain benchmark for screen replay detection.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26855"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Receipt Replay OOD is a small out-of-domain benchmark for screen replay detection. It uses receipts, which share planar geometry, curved corners, wear-and-tear artifacts, and text patterns with identity documents, to evaluate document replay detection models under cross-domain conditions without personally identifiable information constraints.","whyItMatters":"Out-of-domain robustness of screen replay detection remains underexplored, especially under realistic domain shifts. This benchmark provides a public dataset for evaluating generalization across domains, which is critical for deployment in varied presentation attack scenarios.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cd33911ed9917a71957d203e7c32ba1732ef37dc7d56cb538548cfb7b34f3d1d"},"motivation":"Public datasets such as DLC-2021, SynID, and KID34K have significantly contributed to research on presentation attack detection for identity documents, including screen replay attacks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26855","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_poolbench_e79b8c50","familyId":"bmf_9706d0afa65d","name":"PoolBench","oneLine":"PoolBench isolates pooling strategies as the experimental variable in concept representation evaluation for decoder-only LLMs. It covers 17 concepts, 19 pooling strategies, and 3 open-weight models (Llama-3.1-8B, Gemma-2-9B, Mistral-7B) on a corpus of 37,693 real-text passages, with primary axis linear separability (AUROC) and diagnostic axes for steering and disentanglement.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.05162","pdf":"https://arxiv.org/pdf/2608.05162","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.05162"},"evidence":{"snippet":"We introduce PoolBench, a benchmark that isolates pooling as the experimental variable under a fixed evaluation protocol.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.05162"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PoolBench isolates pooling strategies as the experimental variable in concept representation evaluation for decoder-only LLMs. It covers 17 concepts, 19 pooling strategies, and 3 open-weight models (Llama-3.1-8B, Gemma-2-9B, Mistral-7B) on a corpus of 37,693 real-text passages, with primary axis linear separability (AUROC) and diagnostic axes for steering and disentanglement.","whyItMatters":"Pooling is a consequential but under-examined design choice in concept representation work, yet no shared protocol exists for comparing pooling rules across concepts, models, and tasks. PoolBench provides a controlled protocol with released corpus, pre-extracted activations, and scoring code to enable principled comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b1f2e70efdb1ba23147bcbbf582544f71de38918836a8857adc217b0714068bb"},"motivation":"Pooling is a consequential but under-examined design choice in decoder-only concept representation work: practitioners must collapse token-level hidden states into a passage-level vector, yet no shared protocol exists for comparing this choice across concepts, models, and tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.05162","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_persliteval_aca10df9","familyId":"bmf_93ba18bfa5d3","name":"PersLitEval","oneLine":"PersLitEval is a benchmark of 4,514 Persian literature multiple-choice questions across eight categories including spelling, literary devices, grammar, vocabulary, word formation, and conceptual understanding, sourced from Konkur university entrance exam materials.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27015","pdf":"https://arxiv.org/pdf/2605.27015","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27015"},"evidence":{"snippet":"We introduce PersLitEval, a benchmark of 4,514 Persian literature multiple-choice questions across eight fine-grained categories spanning spelling, literary devices, grammar, vocabulary, word formation, and conceptual understanding, sourced from materials for the Konkur university entrance examination.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27015"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"PersLitEval is a benchmark of 4,514 Persian literature multiple-choice questions across eight categories including spelling, literary devices, grammar, vocabulary, word formation, and conceptual understanding, sourced from Konkur university entrance exam materials.","whyItMatters":"LLMs remain poorly evaluated on literary knowledge in non-English languages. PersLitEval provides a fine-grained evaluation of Persian literary understanding, revealing disparities across task difficulty and prompting strategies, which can guide improvements for multilingual literary competence.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e7fb8c408f5b29e53d9708c7e4f48b22f2226d11bcd0582c6fe5c17bef2dbf33"},"motivation":"Despite impressive multilingual capabilities, large language models (LLMs) remain poorly evaluated on literary knowledge in non-English languages.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27015","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_omniinteract_26f9ce32","familyId":"bmf_2f71b445271d","name":"OmniInteract","oneLine":"OmniInteract is a streaming benchmark for real-time omnimodal LLMs evaluated through native online inference over audio-visual streams. It contains 250 videos with 1,430 temporally grounded response slots (1Q1A and 1QnA), with each slot including trigger, response window, and target answer. Metrics include IA-QTF1, Interruption Diagnostic Suite, and Nested Chain Completion Score.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26485","pdf":"https://arxiv.org/pdf/2605.26485","project":null,"code":"https://github.com/Lucky-Lance/OmniInteract","data":null,"hfPaper":"https://huggingface.co/papers/2605.26485"},"evidence":{"snippet":"We introduce OmniInteract, a streaming benchmark for real-time omnimodal large language models evaluated through native online inference over audio-visual streams.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":3,"hfDailySubmittedAt":"2026-05-29T00:00:00.000Z","githubStars":19,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26485"},"ranking":{"90d":{"score":64,"rank":104,"coverage":0.7,"confidence":"Medium"}},"description":"OmniInteract is a streaming benchmark for real-time omnimodal LLMs evaluated through native online inference over audio-visual streams. It contains 250 videos with 1,430 temporally grounded response slots (1Q1A and 1QnA), with each slot including trigger, response window, and target answer. Metrics include IA-QTF1, Interruption Diagnostic Suite, and Nested Chain Completion Score.","whyItMatters":"Real-time omnimodal assistants must process streaming audio-visual input and decide whether and when to respond without access to future content, a capability not captured by offline video benchmarks. OmniInteract provides a native streaming evaluation protocol, revealing that current models remain weak in streaming interaction.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5359f57021ec3668aa29ca67f89097c943ae5ab02aefe65d66d7c54154e74079"},"motivation":"We introduce OmniInteract, a streaming benchmark for real-time omnimodal large language models evaluated through native online inference over audio-visual streams.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26485","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MMLab CUHK","organizationType":"academic-lab","sourceUrl":"https://github.com/Lucky-Lance/OmniInteract","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception","Mathematics & Formal Sciences"],"domainScope":"general"},{"id":"bm_mtm-bench_e1a4ed71","familyId":"bmf_a0d4dfc5a508","name":"MTM-Bench","oneLine":"MTM-Bench is a controlled benchmark for language-conditioned task execution in multilingual settings, enumerating all 27 instruction-content-response language triplets across English, Spanish, and Chinese. It contains 2,430 instances per model across semantic reversal, final-state extraction, and language purity tasks, with decomposed metrics for semantic correctness, language adherence, constraint satisfaction, and joint success.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27649","pdf":"https://arxiv.org/pdf/2605.27649","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27649"},"evidence":{"snippet":"We introduce MTM-Bench, a controlled benchmark for language-conditioned task execution in which each instance is defined by a triplet \\((L_{\\text{instr}}, L_{\\text{content}}, L_{\\text{resp}})\\).","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27649"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"MTM-Bench is a controlled benchmark for language-conditioned task execution in multilingual settings, enumerating all 27 instruction-content-response language triplets across English, Spanish, and Chinese. It contains 2,430 instances per model across semantic reversal, final-state extraction, and language purity tasks, with decomposed metrics for semantic correctness, language adherence, constraint satisfaction, and joint success.","whyItMatters":"Multilingual LLMs are used when instruction, source content, and response languages differ, yet existing evaluations rarely isolate these roles. MTM-Bench provides a fully crossed design to attribute degradation to specific language roles, revealing that response-slot mismatch drives most performance loss and that mismatch count is not a monotonic predictor of difficulty.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"753bd38a7699932ff39b720a13aa24419a9b3088915cdaae11a259e0b053709b"},"motivation":"Multilingual LLMs are increasingly used when instruction, source content, and required response languages do not coincide.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27649","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_memfail_db6b29b0","familyId":"bmf_c88fa7965f13","name":"MemFail","oneLine":"Diagnostic benchmark isolating failure modes of LLM memory systems by evaluating summarization, storage, and retrieval operations.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26667","pdf":"https://arxiv.org/pdf/2605.26667","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.26667"},"evidence":{"snippet":"We introduce MemFail, a diagnostic benchmark that isolates the failure modes of modern LLM memory systems.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26667"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Diagnostic benchmark isolating failure modes of LLM memory systems by evaluating summarization, storage, and retrieval operations.","whyItMatters":"Existing memory benchmarks report aggregate QA accuracy, failing to attribute errors to specific system components. MemFail provides fine-grained diagnostics for memory system design.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"69c64a9638dd7f535ba6c023ef9e4962fb1eed32907a3c419885b0517366e7e1"},"motivation":"Large language model (LLM) agents increasingly rely on external memory systems to remain consistent across long-horizon interactions, but little empirical work has been done to understand the specific failure modes and design choices that these systems present.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26667","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_logdx-ci_ff295ed3","familyId":"bmf_cb2e3cced8f6","name":"LogDx-CI","oneLine":"Benchmark evaluating 11 log reduction tools on 35 GitHub Actions failure cases, scored by 3 LLM debugger families.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.28876","pdf":"https://arxiv.org/pdf/2605.28876","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.28876"},"evidence":{"snippet":"We introduce LogDx-CI, a benchmark that compares 11 context-reduction tools (raw, tail, grep, three RTK modes, two real LLM map-reduce summarizers, three hybrid routers) on 35 real GitHub Actions failure cases, scored by 3 LLM debugger families (Claude Haiku 4.5, Claude Sonnet 4.6, OpenAI gpt-5-mini) plus a Sonnet 4.6 tool-using agent.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.28876"},"ranking":{"90d":{"score":50,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Benchmark evaluating 11 log reduction tools on 35 GitHub Actions failure cases, scored by 3 LLM debugger families.","whyItMatters":"No public comparison existed for which log reductions preserve diagnostic evidence for LLM-based root-cause diagnosis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a17083f3be20a107653db3c4ee28e0dbe9d3f937bc5450b997dab3955f09ee2a"},"motivation":"CI failure logs are large (median 5k lines, max 200k in this corpus) and noisy.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.28876","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LogDx-CI Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2605.28876","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_livek12bench_56b774f5","familyId":"bmf_80ca011e62fa","name":"LiveK12Bench","oneLine":"Dynamic benchmark for multimodal reasoning on high school exam questions in math, physics, chemistry, and biology, with a mock exam evaluation scheme.","area":"Multimodal","applicationDomains":["Science & Research"],"primaryDomain":"Science & Research","industrySectors":["Materials & Chemicals"],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26781","pdf":"https://arxiv.org/pdf/2605.26781","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.26781"},"evidence":{"snippet":"To address these issues, we introduce LiveK12Bench, a dynamic, holistic, multi-disciplinary benchmark designed to evaluate the reasoning abilities of LMMs in realistic examination scenarios.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26781"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Dynamic benchmark for multimodal reasoning on high school exam questions in math, physics, chemistry, and biology, with a mock exam evaluation scheme.","whyItMatters":"Static benchmarks are prone to contamination and fail to reflect real exam constraints. LiveK12Bench provides a growing, realistic evaluation for educational readiness.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a89c4dd2104625b9a43335eb8c12537263c7c9dc148e5fff33ed13594da1392a"},"motivation":"Advanced Large Multimodal Models (LMMs) have demonstrated impressive performance in K-12 reasoning tasks, exhibiting great promise as intelligent tutors.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26781","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"LiveK12Bench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2605.26781","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_ipibench_3c35c7c4","familyId":"bmf_7ebfbd9fee5c","name":"IPIBench","oneLine":"Benchmark for interactive proactive intelligence of MLLMs under streaming video, covering proactive monitoring, task management, and interleaved requests.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27074","pdf":"https://arxiv.org/pdf/2605.27074","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27074"},"evidence":{"snippet":"To address this gap, we introduce IPIBench, the first benchmark for evaluating Interactive Proactive Intelligence of MLLMs under streaming video settings.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27074"},"ranking":{"90d":{"score":46,"rank":null,"coverage":0.15,"confidence":"Low"}},"description":"Benchmark for interactive proactive intelligence of MLLMs under streaming video, covering proactive monitoring, task management, and interleaved requests.","whyItMatters":"Existing benchmarks overlook dynamic multi-turn proactive interactions. IPIBench fills this gap for streaming assistant evaluation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9f7cf8d7b8ba4c0e2f0251e8167e42d6910bd50f145473c1950a5047f9933658"},"motivation":"Recent multimodal large language models (MLLMs) achieve strong performance on reactive question answering, but real-world streaming assistants require proactive reasoning over continuous visual inputs.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27074","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_enpmr-bench_0b8d24ff","familyId":"bmf_d98b3aef5c80","name":"ENPMR-Bench","oneLine":"Benchmark for emotional need-aware proactive memory retrieval in support agents, with 1,800+ dialogues mapped to Maslow's hierarchy.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.27240","pdf":"https://arxiv.org/pdf/2605.27240","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27240"},"evidence":{"snippet":"In this work, we introduce ENPMR-Bench, a benchmark for evaluating Emotional Need-aware Proactive Memory Retrieval (ENPMR), a core capability that enables agents to infer users' latent emotional needs and proactively retrieve appropriate memories to support empathetic interaction.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27240"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Benchmark for emotional need-aware proactive memory retrieval in support agents, with 1,800+ dialogues mapped to Maslow's hierarchy.","whyItMatters":"Current memory retrieval is factual, neglecting emotional needs. ENPMR-Bench evaluates proactive retrieval for empathetic interaction.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"73eb640026de31f2ab73c106d06cca05d1bc96004993c4713310269cac7240bb"},"motivation":"Memory-augmented language agents are increasingly deployed in affective applications such as emotional support, where understanding and responding to users' latent emotional needs is critical.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27240","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_eduvideobench_97f1a2a9","familyId":"bmf_3b433f3c988d","name":"EduVideoBench","oneLine":"Benchmark for educational validity of video generation models, grounded in the Knowledge-Skills-Attitude framework.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.26918","pdf":"https://arxiv.org/pdf/2605.26918","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.26918"},"evidence":{"snippet":"In this work, we present EduVideoBench, the first balanced benchmark in the education domain, grounded in the Knowledge-Skills-Attitude (KSA) framework so that pedagogical adequacy and educational safety are evaluated jointly rather than as ad-hoc quality dimensions.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26918"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Benchmark for educational validity of video generation models, grounded in the Knowledge-Skills-Attitude framework.","whyItMatters":"Existing video benchmarks ignore pedagogical validity, which is critical for classroom use. EduVideoBench fills this gap.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6b1bcd356847d462da4565c563b63171ce4a7d471f2ced9d2e91acd0170e9849"},"motivation":"Video generation models (VGMs) are rapidly entering classrooms, yet existing benchmarks evaluate only perceptual quality, intrinsic faithfulness, generic safety, or video as a reasoning medium, and none assesses whether the outputs are educationally valid.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26918","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_colosseum-v2_833bffbd","familyId":"bmf_0060dc974fcc","name":"Colosseum V2","oneLine":"Simulation benchmark for VLA generalization with 28 tasks across 13 categories and two robot morphologies, using ManiSkill.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["Robotics"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27759","pdf":"https://arxiv.org/pdf/2605.27759","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27759"},"evidence":{"snippet":"To systematically study this gap, we introduce Colosseum V2, a large-scale simulation benchmark for evaluating VLA generalization in robot learning across diverse conditions.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27759"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Simulation benchmark for VLA generalization with 28 tasks across 13 categories and two robot morphologies, using ManiSkill.","whyItMatters":"VLA models often fail under distribution shifts. Colosseum V2 provides standardized in/out-domain evaluation for reproducible comparison.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f91ed8f06b456df929296bd62c360d5f4e98d71329f8ce91c1d3019f874bd626"},"motivation":"Vision-Language-Action (VLA) models demonstrate promising generalization in robotic manipulation, driven by advances in large-scale vision and language pre-training.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27759","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Colosseum V2 Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2605.27759","role":"benchmark-publisher"}],"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_chartact_f82f233a","familyId":"bmf_2274e77eb4da","name":"ChartAct","oneLine":"Interactive benchmark for dynamic chart understanding requiring GUI actions to obtain evidence, with 673 charts and 1,440 QA pairs.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26994","pdf":"https://arxiv.org/pdf/2605.26994","project":null,"code":"https://github.com/wulin-wulin/OSWorld_Chart","data":null,"hfPaper":"https://huggingface.co/papers/2605.26994"},"evidence":{"snippet":"To evaluate this ability, we propose ChartAct, an interactive benchmark for dynamic chart understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":10,"hfDailySubmittedAt":null,"githubStars":13,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26994"},"ranking":{"90d":{"score":64,"rank":106,"coverage":0.7,"confidence":"Medium"}},"description":"Interactive benchmark for dynamic chart understanding requiring GUI actions to obtain evidence, with 673 charts and 1,440 QA pairs.","whyItMatters":"Existing chart benchmarks are static; ChartAct evaluates models on real interactive environments, crucial for real-world chart use.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b0f8226e8171cce7d8050140b4184f3150f85bda90610f55179e81ed9e1af0f4"},"motivation":"Charts are widely used to present complex data for analysis and decision making.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26994","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"ChartAct Team","organizationType":"academic-lab","sourceUrl":"https://github.com/wulin-wulin/OSWorld_Chart","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_assertllm2_9bd7c97e","familyId":"bmf_b6287b8c059f","name":"AssertLLM2","oneLine":"Evaluates LLM generation of SystemVerilog assertions from structured design specifications and RTL, across two tasks: bug-prevention and bug-hunting. The benchmark includes 83 real-world designs with golden and mutated RTL, and assesses syntactic validity, formal provability, coverage, and mutation-based bug detection.","area":"Language & Knowledge","applicationDomains":["Industrial & Engineering"],"primaryDomain":"Industrial & Engineering","industrySectors":["Semiconductors"],"capabilities":[],"topics":["cs.AR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27472","pdf":"https://arxiv.org/pdf/2605.27472","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27472"},"evidence":{"snippet":"To address these limitations, we introduce AssertLLM2, an open-source benchmark for realistic assertion generation in hardware verification.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27472"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"Evaluates LLM generation of SystemVerilog assertions from structured design specifications and RTL, across two tasks: bug-prevention and bug-hunting. The benchmark includes 83 real-world designs with golden and mutated RTL, and assesses syntactic validity, formal provability, coverage, and mutation-based bug detection.","whyItMatters":"Fills a gap in realistic evaluation for assertion generation by using full specifications and buggy RTL, supporting comparison of LLM capabilities for hardware verification tasks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1fab1e04aac1f930b13d9a1b88636933d24d18d71520fe38d92588727e084c71"},"motivation":"Assertion-based verification (ABV) is a cornerstone of modern hardware design, yet manually translating design intent into formal SystemVerilog Assertions (SVAs) remains labor-intensive and error-prone.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27472","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_androiddaily_b00e0630","familyId":"bmf_baa4983aecb6","name":"AndroidDaily","oneLine":"AndroidDaily evaluates mobile GUI agents on 350 daily-use tasks across 94 closed-source Android applications. Automatic scoring is based on a three-tiered guideline system (operational obligations, output quality, negative constraints), with step-level diagnostic judgments.","area":"Agents & Tool Use","applicationDomains":["Consumer & Productivity"],"primaryDomain":"Consumer & Productivity","industrySectors":["Consumer Technology"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-26","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.27761","pdf":"https://arxiv.org/pdf/2605.27761","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.27761"},"evidence":{"snippet":"To bridge this gap, we introduce AndroidDaily, a large-scale benchmark comprising 350 realistic daily-use tasks across 94 high-frequency Android applications spanning transportation, shopping, local services, entertainment, content creation, social media, and everyday utilities.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.27761"},"ranking":{"90d":{"score":null,"rank":null,"coverage":0.0,"confidence":"Low"}},"description":"AndroidDaily evaluates mobile GUI agents on 350 daily-use tasks across 94 closed-source Android applications. Automatic scoring is based on a three-tiered guideline system (operational obligations, output quality, negative constraints), with step-level diagnostic judgments.","whyItMatters":"Fills evaluation gap for real-world closed-source apps where internal states are unavailable, providing a verifiable scoring method based on observable guidelines. Useful for assessing practical deployment of GUI agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0be90bbd75ad5ed9a6df958dc1c7d99f236a605154e9c36040d3f85e8de4c2cd"},"motivation":"The rapid development of GUI foundation models and mobile GUI agents has spurred numerous evaluation benchmarks, yet most rely on simulated environments or open-source applications, leaving real-world closed-source applications largely unevaluated.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.27761","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"specific"},{"id":"bm_wsadbench_d5d448b3","familyId":"bmf_44fa01c8bb2a","name":"WSADBench","oneLine":"WSADBench is a benchmark for weakly supervised anomaly detection (WSAD) that unifies evaluation across incomplete, inexact, and inaccurate supervision scenarios. It evaluates 36 algorithms across 4 modalities (tabular, video, image features, text embeddings) by systematically varying label quantity, granularity, and quality, with protocols and code provided.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2605.26068","pdf":"https://arxiv.org/pdf/2605.26068","project":null,"code":"https://github.com/SUFE-AILAB/WSADBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.26068"},"evidence":{"snippet":"We release WSADBench as an open-source benchmark with code and datasets to facilitate future WSAD research: https://github.com/SUFE-AILAB/WSADBench.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":8,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26068"},"ranking":{},"description":"WSADBench is a benchmark for weakly supervised anomaly detection (WSAD) that unifies evaluation across incomplete, inexact, and inaccurate supervision scenarios. It evaluates 36 algorithms across 4 modalities (tabular, video, image features, text embeddings) by systematically varying label quantity, granularity, and quality, with protocols and code provided.","whyItMatters":"The field of weakly supervised anomaly detection has lacked a unified evaluation framework, with existing benchmarks isolating the three supervision types. WSADBench provides a standardized comparison that reveals performance boundaries across scenarios and informs algorithm selection for practitioners facing limited or noisy labels.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"694a3aefd56c44b4310cd6fb08444abd8c2be2bab95e5d467a32ca978aae6048"},"motivation":"Weakly supervised anomaly detection (WSAD) has developed in three primary directions: incomplete, inexact, and inaccurate supervision.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"KDD 2026 Datasets and Benchmarks Track","evidence":"Accepted at KDD 2026 Datasets and Benchmarks Track","evidenceUrl":"https://arxiv.org/abs/2605.26068","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"KDD 2026 Datasets and Benchmarks Track","reviewStatus":"accepted","decisionRaw":"Accepted at KDD 2026 Datasets and Benchmarks Track","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2605.26068","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at KDD 2026 Datasets and Benchmarks Track","level":"author-claim"}]}],"publishers":[{"name":"SUFE-AILAB","organizationType":"academic-lab","sourceUrl":"https://github.com/SUFE-AILAB/WSADBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_wbench_de4d593c","familyId":"bmf_99d5a953e34c","name":"WBench","oneLine":"WBench evaluates interactive video world models across five dimensions: video quality, setting adherence, interaction adherence, consistency, and physics compliance. It includes 289 test cases and 1,058 multi-turn interaction sequences, covering diverse scenes and control types, with 22 automatic sub-metrics validated against human judgment.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2605.25874","pdf":"https://arxiv.org/pdf/2605.25874","project":"https://meituan-longcat.github.io/WBench/","code":"https://github.com/meituan-longcat/WBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.25874"},"evidence":{"snippet":"To fill this gap, we introduce WBench, a comprehensive multi-turn benchmark for interactive world model evaluation along five dimensions, namely video quality, setting adherence, interaction adherence, consistency, and physics compliance.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":106,"hfDailySubmittedAt":"2026-05-26T00:00:00.000Z","githubStars":213,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25874"},"ranking":{},"description":"WBench evaluates interactive video world models across five dimensions: video quality, setting adherence, interaction adherence, consistency, and physics compliance. It includes 289 test cases and 1,058 multi-turn interaction sequences, covering diverse scenes and control types, with 22 automatic sub-metrics validated against human judgment.","whyItMatters":"No unified standard previously existed for evaluating interactive world models across the required competencies. WBench provides a comprehensive, multi-turn benchmark with a public leaderboard and open data/code, enabling systematic model comparison and diagnostic insights into strengths and weaknesses.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"31e9c387ff6cf694855b2bd000db561961b8980c31b4dac88fe1ec3f7dc74fa8"},"motivation":"Interactive world models are advancing rapidly, yet existing benchmarks cover only part of the required competencies, leaving no unified standard for systematic evaluation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25874","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Meituan LongCat Team","organizationType":"company-research-lab","sourceUrl":"https://github.com/meituan-longcat/WBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_virobench_ca7b2c7e","familyId":"bmf_3d16a3edd82a","name":"ViroBench","oneLine":"ViroBench evaluates nucleotide foundation models on viral genomics tasks across two axes: biological understanding (taxonomy classification, host prediction) and latent biosecurity risk (genome modeling, CDS completion). It includes 18 tasks, 58,314 viral samples, multiple split strategies (genus-disjoint, temporal), and 11 metrics, with a unified evaluation protocol and leaderboard.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.25388","pdf":"https://arxiv.org/pdf/2605.25388","project":null,"code":"https://github.com/QIANJINYDX/ViroBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.25388"},"evidence":{"snippet":"To address this, we introduce ViroBench, the first comprehensive and large-scale benchmark specifically designed for NFMs in viral settings.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":12,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25388"},"ranking":{},"description":"ViroBench evaluates nucleotide foundation models on viral genomics tasks across two axes: biological understanding (taxonomy classification, host prediction) and latent biosecurity risk (genome modeling, CDS completion). It includes 18 tasks, 58,314 viral samples, multiple split strategies (genus-disjoint, temporal), and 11 metrics, with a unified evaluation protocol and leaderboard.","whyItMatters":"Viral genomics lacks a unified evaluation standard for foundation models. ViroBench provides a diagnostic benchmark that measures both task performance and biosecurity risk, enabling model comparison and guiding development of safer, more robust viral nucleotide models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"922d8e86d3c93b75dd8274b046858b8510232f8630b545b8a53029625bb93f02"},"motivation":"Nucleotide sequences constitute the fundamental genetic basis of biological systems, rendering viral genomic analysis critical for biomedical advancement.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25388","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"QIANJINYDX / ViroBench team","organizationType":"community","sourceUrl":"https://github.com/QIANJINYDX/ViroBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_verticue-bench_6f5c5a90","familyId":"bmf_fabfbf0a5160","name":"VertiCue-Bench","oneLine":"VertiCue-Bench evaluates multimodal large language models on geospatial reasoning using canopy height models to resolve 2D ambiguity in remote sensing natural scenes. It includes 1,534 instances across 17 tasks, testing height perception and semantic reasoning.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.25784","pdf":"https://arxiv.org/pdf/2605.25784","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.25784"},"evidence":{"snippet":"To address this gap, we introduce VertiCue-Bench, the first diagnostic benchmark for CHM-grounded geospatial reasoning.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25784"},"ranking":{},"description":"VertiCue-Bench evaluates multimodal large language models on geospatial reasoning using canopy height models to resolve 2D ambiguity in remote sensing natural scenes. It includes 1,534 instances across 17 tasks, testing height perception and semantic reasoning.","whyItMatters":"Current remote sensing benchmarks are mostly 2D-centric, failing in environments with spectral confusion. This benchmark addresses the gap of whether models can leverage vertical cues for semantic disambiguation, providing insights into geometry-to-semantics reasoning in MLLMs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e43330584dc237458c09f246ff95527fc3ae128e63af3a1dc78cbe71fd7b585d"},"motivation":"Multimodal Large Language Models (MLLMs) have recently shown promising progress in geospatial reasoning.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25784","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_uav-ovo_af2dbcd6","familyId":"bmf_90a06c92b3f9","name":"UAV-OVO","oneLine":"UAV-OVO is an out-of-viewpoint generalization benchmark for UAV action recognition. It evaluates models on low-depression viewpoint training data and tests on high-depression viewpoint out-of-distribution data, with class-matched ID/OOD splits.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.25615","pdf":"https://arxiv.org/pdf/2605.25615","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.25615"},"evidence":{"snippet":"We introduce UAV-OVO, an Out-of-Viewpoint generalization benchmark for UAV action recognition.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25615"},"ranking":{},"description":"UAV-OVO is an out-of-viewpoint generalization benchmark for UAV action recognition. It evaluates models on low-depression viewpoint training data and tests on high-depression viewpoint out-of-distribution data, with class-matched ID/OOD splits.","whyItMatters":"Standard UAV action recognition benchmarks often overlook viewpoint shifts, leading to models that rely on viewpoint-specific shortcuts. UAV-OVO provides a controlled testbed to measure and improve robustness to viewpoint changes, which is critical for real-world UAV deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d17b74d39b18405218847cb9482d903f3d40c346ac5d1150fd7886bcc3c7dece"},"motivation":"UAV action recognition faces a deployment shift that standard benchmarks often obscure: a model trained on UAV footage captured from low-depression viewpoints may be required to recognize the same action classes from high-depression viewpoints.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25615","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_streamprofilebench_ccd7b568","familyId":"bmf_5cbc6af978f5","name":"StreamProfileBench","oneLine":"StreamProfileBench evaluates LLMs on fine-grained streaming user profiling. Models maintain a rolling persona summary from a stream of user posts and predict which tags from a candidate pool the user will engage with next. Includes over 120,000 posts from 7,000+ users across five Chinese platforms with metrics for recall, novelty, stability, and error rates.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.25758","pdf":"https://arxiv.org/pdf/2605.25758","project":null,"code":"https://github.com/WaterWang-001/StreamProfileBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.25758"},"evidence":{"snippet":"To bridge this gap, we introduce StreamProfileBench, a large-scale benchmark for fine-grained streaming user profiling.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25758"},"ranking":{},"description":"StreamProfileBench evaluates LLMs on fine-grained streaming user profiling. Models maintain a rolling persona summary from a stream of user posts and predict which tags from a candidate pool the user will engage with next. Includes over 120,000 posts from 7,000+ users across five Chinese platforms with metrics for recall, novelty, stability, and error rates.","whyItMatters":"Existing user profiling benchmarks use static data, failing to capture real-world streaming UGC and rapidly evolving interests. StreamProfileBench provides a dynamic evaluation that measures plasticity-stability balance, revealing conservative bias in LLMs and enabling practical improvements for personalized systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bb535c7a9f14f0b08f2e3098bc7a1a486aa3380373a7bad8c12425f61406dbd5"},"motivation":"Large Language Models (LLMs) have reshaped user profiling, yet current evaluations mainly focus on static data snapshots.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25758","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"StreamProfileBench Team","organizationType":"community","sourceUrl":"https://github.com/WaterWang-001/StreamProfileBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_spapath-bench_cda2e02f","familyId":"bmf_3601f520ca7d","name":"SpaPath-Bench","oneLine":"SpaPath-Bench evaluates pathology foundation models on spatial domain identification using paired whole slide images and spatial transcriptomics data from 42 public slides, measuring partition quality via unsupervised spatial coherence, transcriptomics-referenced agreement, and expert-referenced agreement.","area":"Vision & 3D","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.25764","pdf":"https://arxiv.org/pdf/2605.25764","project":"https://bokai-zhao.github.io/SpaPath-benchboard/","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.25764"},"evidence":{"snippet":"We present SpaPath-Bench, a representation level benchmark designed to diagnose spatial representation capability in PFMs.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25764"},"ranking":{},"description":"SpaPath-Bench evaluates pathology foundation models on spatial domain identification using paired whole slide images and spatial transcriptomics data from 42 public slides, measuring partition quality via unsupervised spatial coherence, transcriptomics-referenced agreement, and expert-referenced agreement.","whyItMatters":"Standard task-level endpoints obscure what pathology embeddings encode about tissue spatial structure. SpaPath-Bench provides a representation-level diagnostic that isolates spatial understanding, enabling model developers to compare encoders and methods on a fixed protocol and choose architectures suited for spatially aware computational pathology.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3ae6a971492f16861805e83eae3585e3ce883b02ceb4064c6d049eb5c0a403cd"},"motivation":"Pathology foundation models (PFMs) have emerged as a core approach for learning transferable representations from whole slide images (WSIs), and they are typically benchmarked through downstream clinical endpoints.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25764","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"SpaPath-Bench team","organizationType":"academic-lab","sourceUrl":"https://bokai-zhao.github.io/SpaPath-benchboard/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_repomirage_322a2daa","familyId":"bmf_defa1b39bb7c","name":"RepoMirage","oneLine":"RepoMirage is a two-stage evaluation suite built on SWE-Bench Verified that applies semantics-preserving repository-level perturbations and extended tasks to probe repository context reasoning in code agents.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26177","pdf":"https://arxiv.org/pdf/2605.26177","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.26177"},"evidence":{"snippet":"To investigate this question, we introduce RepoMirage, a two-stage evaluation suite built on SWE-Bench Verified that adopts perturbation as a diagnostic tool to increase the demand for context reasoning by transforming how the repository is exposed.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26177"},"ranking":{},"description":"RepoMirage is a two-stage evaluation suite built on SWE-Bench Verified that applies semantics-preserving repository-level perturbations and extended tasks to probe repository context reasoning in code agents.","whyItMatters":"It aims to isolate repository context reasoning from end-to-end issue resolution performance, revealing a gap that could inform structure-aware agent design.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cad8c803504208bfc383b6e69168d3ddfababa4d8939051acf7b1c39d5ced261"},"motivation":"Code agents are currently having skillful performance on repository-level software engineering benchmarks, but it remains unclear whether success on end-to-end tasks such as issue resolution truly reflects repository context reasoning, the ability to identify the task-relevant information across multiple files and reason over the relations among them.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26177","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_pitchbench_7fffde93","familyId":"bmf_ccf6122c0774","name":"PitchBench","oneLine":"PitchBench evaluates pitch hearing in audio-language models across 28 experiments spanning absolute and relative pitch perception in sequences and chords, varying acoustic conditions and response formats.","area":"Speech & Audio","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.SD"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26176","pdf":"https://arxiv.org/pdf/2605.26176","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.26176"},"evidence":{"snippet":"We introduce PitchBench, an evaluation suite that systematically measures pitch hearing in ALMs.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26176"},"ranking":{},"description":"PitchBench evaluates pitch hearing in audio-language models across 28 experiments spanning absolute and relative pitch perception in sequences and chords, varying acoustic conditions and response formats.","whyItMatters":"Pitch perception is foundational for musical reasoning, yet existing benchmarks probe it indirectly. PitchBench provides a systematic, controlled evaluation to identify limitations in current models and support the development of pitch-aware audio-language systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3adb7e98ef93ba2631d3d5782ce14338969a9f02f423d508df118dece5bc5f18"},"motivation":"Audio-language models (ALMs) are increasingly used in real-world applications that require understanding music, from music tutoring and transcription to captioning, recommendation systems, and music production.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26176","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_permembench_15a1b01c","familyId":"bmf_fb7852f323c5","name":"PerMemBench","oneLine":"PerMemBench evaluates personalized memory systems for LLM agents using multi-year, multi-domain interaction histories across 20 user personas, measuring memory retention accuracy.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.25535","pdf":"https://arxiv.org/pdf/2605.25535","project":null,"code":"https://github.com/yeonjun-in/PerMemBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.25535"},"evidence":{"snippet":"We introduce PerMemBench, the first benchmark for evaluating personalized memory systems, featuring multi year, multi domain interaction histories across diverse user personas.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":45,"hfDailySubmittedAt":"2026-05-26T00:00:00.000Z","githubStars":10,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25535"},"ranking":{},"description":"PerMemBench evaluates personalized memory systems for LLM agents using multi-year, multi-domain interaction histories across 20 user personas, measuring memory retention accuracy.","whyItMatters":"Universal memory policies waste budget on transient interactions and fail to preserve critical context. PerMemBench enables evaluation of personalization, revealing that accurate gating remains an open challenge.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f7bcccbe7d923b8a2bbed3c622d775edc3b6607c3633cd1a06c98d82b39db8b0"},"motivation":"Existing large language model (LLM) based memory systems apply universal, static policies that overlook a fundamental reality: the contexts that are worth storing in memory are different across users.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25535","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"PerMemBench team","organizationType":"academic-lab","sourceUrl":"https://github.com/yeonjun-in/PerMemBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_omnitom_56af1224","familyId":"bmf_e7330cebd753","name":"OmniToM","oneLine":"OmniToM evaluates theory of mind in LLMs by requiring explicit belief modeling, extracting belief propositions and labeling them with seven-dimensional schema labels across 895 stories.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26322","pdf":"https://arxiv.org/pdf/2605.26322","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.26322"},"evidence":{"snippet":"In order to address this research gap, we introduce OmniToM, a benchmark that directly evaluates these representations by requiring explicit modeling of belief structures for all relevant actors within a narrative.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26322"},"ranking":{},"description":"OmniToM evaluates theory of mind in LLMs by requiring explicit belief modeling, extracting belief propositions and labeling them with seven-dimensional schema labels across 895 stories.","whyItMatters":"It addresses the gap of end-point question answering in ToM evaluation by forcing explicit mental-state representation, potentially revealing actor-specific belief-tracking bottlenecks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"87223a18c750df8b2885abbbc7db0603d64c86f3bc7228e21475900154520c8c"},"motivation":"Theory of Mind (ToM), the ability to infer others' knowledge, intentions, and emotions, is commonly evaluated in large language models (LLMs) using end-point question answering, where performance is judged solely by the final answer to a social reasoning query.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26322","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_multiseismo_9617a097","familyId":"bmf_e1646a92b259","name":"MULTISEISMO","oneLine":"MultiSeismo is a multimodal seismic dataset with over 16K events integrating waveforms, intensity maps, population exposure, and text, plus MISCE instruction set for seismic reasoning tasks like retrieval and cross-modal analysis.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.26320","pdf":"https://arxiv.org/pdf/2605.26320","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.26320"},"evidence":{"snippet":"These results prove that MultiSeismo provides a rigorous benchmark for future multimodal research in seismology and validate the success of our domain specific architectural adaptations.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26320"},"ranking":{},"description":"MultiSeismo is a multimodal seismic dataset with over 16K events integrating waveforms, intensity maps, population exposure, and text, plus MISCE instruction set for seismic reasoning tasks like retrieval and cross-modal analysis.","whyItMatters":"It enables evaluation of multimodal models on specialized scientific data, highlighting challenges in time-series processing and supporting development of domain-specific models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cb486fc30dd6a8f9883f28d18e07c78765e68ae64ff6bdd71a5944140c5ceaf7"},"motivation":"The application of generalist multimodal models (GMMs) to specialized scientific domains remains limited due to the scarcity of comprehensive domain-specific datasets that integrate multiple data modalities beyond text and images.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26320","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_mm-creativitybench_e366f23e","familyId":"bmf_534ca0e1c4b9","name":"MM-CreativityBench","oneLine":"MM-CreativityBench evaluates large multimodal models on affordance-grounded creative tool use, requiring scene inspection, entity/part selection, and physically feasible solutions in visually rich environments.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Tool use"],"topics":["Agents","Multimodal"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26396","pdf":"https://arxiv.org/pdf/2605.26396","project":null,"code":"https://github.com/CreativityBench/MM-CreativityBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.26396"},"evidence":{"snippet":"To evaluate this ability, we introduce MM-CreativityBench, a benchmark for affordance-grounded creative tool use in visually rich, physically constrained environments.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":21,"hfDailySubmittedAt":"2026-05-28T00:00:00.000Z","githubStars":4,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26396"},"ranking":{},"description":"MM-CreativityBench evaluates large multimodal models on affordance-grounded creative tool use, requiring scene inspection, entity/part selection, and physically feasible solutions in visually rich environments.","whyItMatters":"It probes beyond pattern recognition, assessing grounded exploration and reasoning. Current models show gaps in exploration and hallucination, motivating preference-based alignment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d54abdaedabbfb3704140577574bd4b6c6df625f528f74f7076565e78c9f6678"},"motivation":"Large multimodal models (LMMs) have rapidly advanced in perception and reasoning; however, it remains unclear whether these capabilities generalize to discovering visually grounded solutions in open-ended environments, beyond pattern recognition.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26396","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MM-CreativityBench team","organizationType":"academic-lab","sourceUrl":"https://github.com/CreativityBench/MM-CreativityBench","role":"benchmark-publisher"}],"capabilityGroups":["Agents","Tool Calling"],"domainScope":"general"},{"id":"bm_metaphorvu-bench_ab9e08cf","familyId":"bmf_2f1827922297","name":"MetaphorVU-Bench","oneLine":"MetaphorVU-Bench evaluates metaphorical video understanding in multimodal LLMs through tasks requiring cross-domain mapping, with a benchmark dataset and evaluation code publicly available.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.25461","pdf":"https://arxiv.org/pdf/2605.25461","project":null,"code":"https://github.com/icip-cas/MetaphorVU","data":null,"hfPaper":"https://huggingface.co/papers/2605.25461"},"evidence":{"snippet":"To bridge this gap, we propose MetaphorVU-Bench, the first systematic and comprehensive benchmark dedicated to metaphorical video understanding.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":"2026-05-26T00:00:00.000Z","githubStars":10,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25461"},"ranking":{},"description":"MetaphorVU-Bench evaluates metaphorical video understanding in multimodal LLMs through tasks requiring cross-domain mapping, with a benchmark dataset and evaluation code publicly available.","whyItMatters":"It fills the gap of systematic evaluation of high-order cognitive capabilities in video understanding, revealing defects in cross-domain mapping and enabling future improvements.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d8c0f822677392ee51b5ed37bebe65731cf5117faab044fd4dd0e314e22ce090"},"motivation":"Metaphorical videos are prevalent across various real-world scenarios to convey complex ideas, and understanding them typically requires high-order cognitive capabilities.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25461","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"MetaphorVU team","organizationType":"academic-lab","sourceUrl":"https://github.com/icip-cas/MetaphorVU","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_longav-compass_f7e880eb","familyId":"bmf_67cc07e08a0c","name":"LongAV-Compass","oneLine":"LongAV-Compass evaluates minute-scale audio-visual generation across text-to-audio-video, image-to-audio-video, and video-to-audio-video tasks. It includes 284 curated test cases and an evaluation framework combining multimodal metrics with MLLM-assisted assessment across 20+ dimensions covering segment quality, cross-segment consistency, narrative coherence, semantic alignment, and audiovisual synchronization.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26244","pdf":"https://arxiv.org/pdf/2605.26244","project":null,"code":"https://github.com/pkucs-Ltf/LongAV-Compass","data":null,"hfPaper":"https://huggingface.co/papers/2605.26244"},"evidence":{"snippet":"To bridge this gap, we introduce LongAV-Compass, a systematic benchmark for minute-long audio-visual generation.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":38,"hfDailySubmittedAt":"2026-05-27T00:00:00.000Z","githubStars":17,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26244"},"ranking":{},"description":"LongAV-Compass evaluates minute-scale audio-visual generation across text-to-audio-video, image-to-audio-video, and video-to-audio-video tasks. It includes 284 curated test cases and an evaluation framework combining multimodal metrics with MLLM-assisted assessment across 20+ dimensions covering segment quality, cross-segment consistency, narrative coherence, semantic alignment, and audiovisual synchronization.","whyItMatters":"Existing audio-visual evaluation is limited to short clips, leaving a gap for minute-scale generation. LongAV-Compass provides a standardized protocol for diagnosing degradation in identity consistency, narrative coherence, and alignment over long horizons, supporting comparison of long-form generation models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"903d8a598d71a3ad64fe4c5d7fe6a2514e7320ca4ad22b1e9d5c1061ad0eb1d6"},"motivation":"Audio-visual generation is rapidly advancing from short clips to minute-long content, while existing evaluation protocols remain largely confined to short-form settings.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26244","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"PKU CS LongAV Group","organizationType":"academic-lab","sourceUrl":"https://github.com/pkucs-Ltf/LongAV-Compass","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_kidbench_e196ca1f","familyId":"bmf_d5c09a3ec477","name":"KIDBench","oneLine":"KIDBench evaluates child-facing safety of LLMs for ages 7-11 using realistic queries and multi-turn child-actor simulations, scored by an LLM-as-a-Judge rubric.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.25510","pdf":"https://arxiv.org/pdf/2605.25510","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.25510"},"evidence":{"snippet":"We introduce KIDBench, a benchmark for evaluating child-facing LLM safety for ages 7-11 using a developmental-psychology-grounded LLM-as-a-Judge rubric.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25510"},"ranking":{},"description":"KIDBench evaluates child-facing safety of LLMs for ages 7-11 using realistic queries and multi-turn child-actor simulations, scored by an LLM-as-a-Judge rubric.","whyItMatters":"It addresses a gap in LLM safety evaluation by focusing on age-appropriate responses for children, with a novel rubric and multi-turn assessment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b9f62aa683b7a3015a20ce3331eeaac39dc91d4faa87f4b9ea92a774f67e38e7"},"motivation":"Children increasingly have access to Large Language Models (LLMs), which may expose them to responses that are developmentally inappropriate or require age-sensitive safety, guidance, and boundaries.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25510","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_floatbench_4e9fbef2","familyId":"bmf_04b77197562e","name":"FLOATBench","oneLine":"FLOATBench is a tabular benchmark for surrogate modeling of tower fatigue in 22 MW floating offshore wind turbines. It provides 582,120 per-section fatigue damage labels from OpenFAST simulations across three tower geometries, with regime-aware splits and three evaluation protocols (random, within-tower regime-aware, cross-tower transfer).","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.25717","pdf":"https://arxiv.org/pdf/2605.25717","project":null,"code":"https://github.com/Joao97ribeiro/FLOATBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.25717"},"evidence":{"snippet":"We present FLOATBench, a public tabular benchmark with $582{,}120$ per-section fatigue-damage labels across three $22$ MW FOWT tower geometries, derived from $19{,}404$ high-fidelity OpenFAST simulations across the three towers ($6{,}468$ per tower: $1{,}078$ aligned wind/wave operating points $\\times$ six turbulence seeds), labeled at $30$ cross-sections per tower.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25717"},"ranking":{},"description":"FLOATBench is a tabular benchmark for surrogate modeling of tower fatigue in 22 MW floating offshore wind turbines. It provides 582,120 per-section fatigue damage labels from OpenFAST simulations across three tower geometries, with regime-aware splits and three evaluation protocols (random, within-tower regime-aware, cross-tower transfer).","whyItMatters":"Previously, no shared benchmark existed for FOWT fatigue surrogates, hindering comparison. FLOATBench provides a standardized dataset and protocol, enabling fair evaluation and highlighting regime-aware performance differences.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8c08a697b2f308defb6d96675dfff474a0ae6be9f4233f829379ba72711e5bc0"},"motivation":"Most of the world's offshore wind resource lies in waters too deep for fixed-bottom foundations, making floating offshore wind turbines (FOWTs) essential for deep-water deployment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25717","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DeCoDELab","organizationType":"academic-lab","sourceUrl":"https://github.com/Joao97ribeiro/FLOATBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_enterprisemem-bench_96086215","familyId":"bmf_78a3d7491203","name":"EnterpriseMem-Bench","oneLine":"EnterpriseMem-Bench is a multi-turn Text-to-SQL benchmark with 300 sessions and 1,400 turns across three enterprise domains, featuring deterministic ground truth and per-turn memory-critical annotations.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26394","pdf":"https://arxiv.org/pdf/2605.26394","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.26394"},"evidence":{"snippet":"We introduce EnterpriseMem-Bench, a multi-turn Text-to-SQL benchmark of 300 sessions and 1,400 turns built programmatically from three enterprise domains (BIRD financial, SEC EDGAR, Northwind), with deterministic ground truth and per-turn memory-critical annotation.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26394"},"ranking":{},"description":"EnterpriseMem-Bench is a multi-turn Text-to-SQL benchmark with 300 sessions and 1,400 turns across three enterprise domains, featuring deterministic ground truth and per-turn memory-critical annotations.","whyItMatters":"It addresses the lack of multi-turn evaluation in Text-to-SQL, providing insights into memory architecture effects and model performance degradation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"dafbef18e13b541cae79dce70f6abdeb427944fc926fdef83e553e75f706e941"},"motivation":"Multi-turn Text-to-SQL is central to enterprise analytics yet remains predominantly evaluated in single-turn settings.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26394","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_discoverphysics_9ae7e548","familyId":"bmf_48d63b5c443b","name":"DiscoverPhysics","oneLine":"DiscoverPhysics is an interactive benchmark for LLM agents to discover laws of motion in simulated worlds with altered physics, scoring trajectory MSE and explanation quality.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Interactive Environment","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26087","pdf":"https://arxiv.org/pdf/2605.26087","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.26087"},"evidence":{"snippet":"We introduce DiscoverPhysics, an interactive benchmark that asks a LLM agent to discover the laws of motion of a simulated world whose physics deliberately deviates from our own.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26087"},"ranking":{},"description":"DiscoverPhysics is an interactive benchmark for LLM agents to discover laws of motion in simulated worlds with altered physics, scoring trajectory MSE and explanation quality.","whyItMatters":"It probes long-horizon reasoning and hypothesis refinement, distinguishing recall from genuine scientific reasoning.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"17d1615f8ab47ce652f2f499d346be2d6771788446ac84148b256c1addd7b314"},"motivation":"Frontier LLMs now perform strongly across a wide range of physics evaluations, but it is hard to disentangle genuine reasoning from recall of established science.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26087","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_claw-anything_3c56b6b4","familyId":"bmf_ebd37017ecf1","name":"Claw-Anything","oneLine":"Claw-Anything evaluates always-on LLM personal assistants in simulated environments with long-horizon activity histories, interdependent backend services, and GUI/CLI across devices. It includes 200 human-verified tasks scored on completion, robustness, communication, and safety, with a live leaderboard.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26086","pdf":"https://arxiv.org/pdf/2605.26086","project":"https://libercoders.github.io/Claw-Anything/","code":"https://github.com/LiberCoders/Claw-Anything","data":"https://huggingface.co/datasets/LiberCoders/Claw-Anything","hfPaper":"https://huggingface.co/papers/2605.26086"},"evidence":{"snippet":"To address this gap, we introduce Claw-Anything, a benchmark that expands agent context along three dimensions: long-horizon activity histories, interdependent backend services, and integrated GUI and CLI interaction across multiple devices.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-reviewed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":25,"hfDailySubmittedAt":"2026-05-26T00:00:00.000Z","githubStars":48,"githubScope":"benchmark_repo","hfDatasetDownloads":80,"hfDatasetLikes":10},"source":{"type":"arxiv","id":"2605.26086"},"ranking":{},"description":"Claw-Anything evaluates always-on LLM personal assistants in simulated environments with long-horizon activity histories, interdependent backend services, and GUI/CLI across devices. It includes 200 human-verified tasks scored on completion, robustness, communication, and safety, with a live leaderboard.","whyItMatters":"It expands agent evaluation to broad, always-on contexts, revealing capability gaps in stateful, proactive assistance and supporting scalable data generation for training.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a4ba1f383be74e2a945572f6c15d0cad22707c80dfa7a662d4ae181581c0c443"},"motivation":"Large language model agents are increasingly envisioned as always-on personal assistants with access to anything relevant in the user's digital world.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","curation":{"state":"source-reviewed","reviewedAt":"2026-08-19","sources":["https://github.com/LiberCoders/Claw-Anything","https://huggingface.co/datasets/LiberCoders/Claw-Anything","https://arxiv.org/abs/2605.26086"]},"publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26086","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"releaseDates":{"firstPublicAt":"2026-05-25","paperV1At":"2026-05-25"},"publishers":[{"name":"LiberCoders","organizationType":"community","sourceUrl":"https://github.com/LiberCoders/Claw-Anything","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cityrep_fb8ebfd4","familyId":"bmf_443c2445803a","name":"CITYREP","oneLine":"CityRep is a benchmark for urban representation learning across 8 cities and 8 tasks with spatially structured splits to mitigate leakage.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.26036","pdf":"https://arxiv.org/pdf/2605.26036","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.26036"},"evidence":{"snippet":"To address this, we propose CityRep, a unified benchmark that evaluates urban representations across data modalities, cities, and tasks using spatially structured splits.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26036"},"ranking":{},"description":"CityRep is a benchmark for urban representation learning across 8 cities and 8 tasks with spatially structured splits to mitigate leakage.","whyItMatters":"It addresses inconsistencies in urban representation evaluation and supports generalization-aware model selection.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"180885379bfe25d2f368eaa9717ca09a77ea342dd56f3ef7affa362a9fa8110a"},"motivation":"Urban representation learning encodes complex urban environments into general-purpose embeddings for diverse downstream tasks and emerging urban foundation models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26036","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_authtrace_cea4d208","familyId":"bmf_fe5b1fd5a9d7","name":"AuthTrace","oneLine":"AuthTrace is a diagnostic benchmark for evidence construction in thematically dense single-author corpora, with fan-in annotations and pack-level protocol.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.25382","pdf":"https://arxiv.org/pdf/2605.25382","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.25382"},"evidence":{"snippet":"We introduce AuthTrace, a diagnostic benchmark built on thematically dense single-author corpora where near-miss distractors share style, topic, and vocabulary with the required evidence.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25382"},"ranking":{},"description":"AuthTrace is a diagnostic benchmark for evidence construction in thematically dense single-author corpora, with fan-in annotations and pack-level protocol.","whyItMatters":"It provides a diagnostic lens for identifying where evidence construction fails and which paradigm works best.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a947c9fbcad978fba5a4e9fd32cd721cf97546ed3efb60e4488e21e0a145fa54"},"motivation":"Evidence construction--the stage that determines which passages reach the language model before generation begins--is evaluated paradigm by paradigm, leaving practitioners with no principled way to diagnose which organization strategy fails, where, or why.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25382","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_agingbench_6fa9d4b4","familyId":"bmf_425e6cf5e16c","name":"AgingBench","oneLine":"Evaluates the reliability of deployed AI agents over extended operational lifetimes. The benchmark organizes agent aging into four mechanisms—compression, interference, revision, and maintenance—and uses temporal dependency graphs and paired counterfactual probes to produce diagnostic profiles of the memory pipeline's write, retrieval, and utilization stages. It includes multiple scenarios and memory policies, with scoring based on task performance over sessions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.26302","pdf":"https://arxiv.org/pdf/2605.26302","project":null,"code":"https://github.com/VITA-Group/AgingBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.26302"},"evidence":{"snippet":"We introduce AgingBench, a longitudinal reliability benchmark for agent lifespan engineering: measuring not only whether deployed agents degrade, but what form the degradation takes and where repair should target.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":32,"hfDailySubmittedAt":"2026-05-28T00:00:00.000Z","githubStars":15,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.26302"},"ranking":{},"description":"Evaluates the reliability of deployed AI agents over extended operational lifetimes. The benchmark organizes agent aging into four mechanisms—compression, interference, revision, and maintenance—and uses temporal dependency graphs and paired counterfactual probes to produce diagnostic profiles of the memory pipeline's write, retrieval, and utilization stages. It includes multiple scenarios and memory policies, with scoring based on task performance over sessions.","whyItMatters":"Existing agent evaluations are snapshot-based and ignore how agents degrade after deployment. This benchmark measures longevity and provides mechanism-level diagnosis, enabling stage-targeted repair and more dependable agent deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"84f93073e07349155d3e1a6a3f72cecefabf341b07baf03a34b6f6ae578b3542"},"motivation":"Long-lived AI agents are increasingly deployed as persistent operational systems, yet they are still evaluated like freshly initialized models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.26302","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"VITA Group","organizationType":"academic-lab","sourceUrl":"https://github.com/VITA-Group/AgingBench","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_agenthijack_15cbe4d0","familyId":"bmf_48bd013b2a40","name":"AgentHijack","oneLine":"Evaluates the robustness of computer use agents under common environment corruptions such as pop-ups, resolution changes, and competing applications. The benchmark introduces 9 configurable corruptions and evaluates agent performance on desktop tasks using multimodal LLM-based agents, measuring task completion rates.","area":"Agents & Tool Use","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Robustness"],"topics":["Agents"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-25","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.25707","pdf":"https://arxiv.org/pdf/2605.25707","project":"https://AgentHijack.github.io","code":"https://github.com/tmlr-group/AgentHijack","data":null,"hfPaper":"https://huggingface.co/papers/2605.25707"},"evidence":{"snippet":"We introduce AgentHijack, a benchmark designed to evaluate the robustness of computer-use agents under common corruptions, where the uncertainties in dynamic environment disrupt the execution flow without direct adversarial intent.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":6,"hfDailySubmittedAt":"2026-05-28T00:00:00.000Z","githubStars":6,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25707"},"ranking":{},"description":"Evaluates the robustness of computer use agents under common environment corruptions such as pop-ups, resolution changes, and competing applications. The benchmark introduces 9 configurable corruptions and evaluates agent performance on desktop tasks using multimodal LLM-based agents, measuring task completion rates.","whyItMatters":"Real-world execution environments are imperfect, and minor corruptions can cause significant performance degradation. This benchmark quantifies agent fragility and supports the development of more robust computer use agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"8eec12423b8a25aee11bc508fa90f3d94f57674ede97c647b5518cf304495aa3"},"motivation":"Autonomous computer use agents that powered by multimodal large language models (MLLMs) are emerging as capable assistants for completing complex digital workflows.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026","evidence":"accepted by ICML 2026","evidenceUrl":"https://arxiv.org/abs/2605.25707","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026","reviewStatus":"accepted","decisionRaw":"accepted by ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2605.25707","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"accepted by ICML 2026","level":"author-claim"}]}],"publishers":[{"name":"AgentHijack Team","organizationType":"academic-lab","sourceUrl":"https://github.com/tmlr-group/AgentHijack","role":"benchmark-publisher"}],"capabilityGroups":["Agents"],"domainScope":"general"},{"id":"bm_realbench_3d168523","familyId":"bmf_f0848f948867","name":"RealBench","oneLine":"Evaluates data-driven numerical weather forecasting models under operational conditions, using strictly out-of-distribution test data from 2025 and integrating low-latency operational analysis and large-scale in-situ observations from over 10,000 stations. It provides metrics for global forecasting and for extreme events such as heatwaves, cold surges, and tropical cyclones.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24945","pdf":"https://arxiv.org/pdf/2605.24945","project":null,"code":"https://github.com/lixruize-del/NWP-Benchmark","data":null,"hfPaper":"https://huggingface.co/papers/2605.24945"},"evidence":{"snippet":"In this work, we introduce RealBench, a next-generation benchmark for AI weather forecasting that emphasizes realistic evaluation under operational conditions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":7,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24945"},"ranking":{},"description":"Evaluates data-driven numerical weather forecasting models under operational conditions, using strictly out-of-distribution test data from 2025 and integrating low-latency operational analysis and large-scale in-situ observations from over 10,000 stations. It provides metrics for global forecasting and for extreme events such as heatwaves, cold surges, and tropical cyclones.","whyItMatters":"Existing benchmarks rely on reanalysis products that do not reflect real-time operational constraints, leading to mismatches between benchmark scores and real-world performance. This benchmark provides a more faithful and operationally relevant evaluation paradigm.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"09b5679de8cfa25c72850359b6871f73ef98c3bab38d223f8171a79b473b10c5"},"motivation":"Accurate evaluation of weather forecasting models is critical for their reliable deployment in real-world applications.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24945","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"NWP-Benchmark contributors","organizationType":"academic-lab","sourceUrl":"https://github.com/lixruize-del/NWP-Benchmark","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_judgmentbench_729b2b9c","familyId":"bmf_4b9658370bd4","name":"JudgmentBench","oneLine":"A dataset of 30 legal tasks with rubric scores and pairwise preference judgments from practicing attorneys, used to compare rubric-based scoring and comparative judgment.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-05-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.25240","pdf":"https://arxiv.org/pdf/2605.25240","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.25240"},"evidence":{"snippet":"We release JudgmentBench, a benchmark of 30 real-world legal tasks, paired with 1,539 rubric scores and 1,530 pairwise preference judgments collected from practicing attorneys--including at major U.S.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25240"},"ranking":{},"description":"A dataset of 30 legal tasks with rubric scores and pairwise preference judgments from practicing attorneys, used to compare rubric-based scoring and comparative judgment.","whyItMatters":"Supports research on expert judgment elicitation and aggregation in domains without ground truth.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6b8af1641e28f6c294d26b4cb93e1748d5c39742fa028ea514761628a1273283"},"motivation":"Two methodologies dominate current practices of benchmarking: rubric-based scoring evaluates items against predefined criteria, whereas comparative judgment elicits pairwise preferences between outputs.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25240","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_grouptravelbench_058d38b0","familyId":"bmf_014c2d359ad4","name":"GroupTravelBench","oneLine":"A benchmark for multi-user, multi-turn travel planning with 650 tasks and a synchronous group-chat sandbox, evaluating elicitation, coordination, and fairness-aware planning.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Planning"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-24","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.25200","pdf":"https://arxiv.org/pdf/2605.25200","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.25200"},"evidence":{"snippet":"To bring the task back to its multi-user reality, we introduce \\textbf{\\textit{GroupTravelBench}}, the first benchmark for \\textbf{multi-user, multi-turn} travel planning.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25200"},"ranking":{},"description":"A benchmark for multi-user, multi-turn travel planning with 650 tasks and a synchronous group-chat sandbox, evaluating elicitation, coordination, and fairness-aware planning.","whyItMatters":"Highlights the challenge of group-level outcome quality for LLM agents.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"44f977987f1b71d55d5c2296f171786dc03390dd7f2adcbe05fa246cdbd9ed3d"},"motivation":"Travel planning in the real world is overwhelmingly a \\textit{group} activity, yet existing LLM travel-planning benchmarks reduce it to a single user, where the field is approaching saturation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25200","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_clueaegis-bench_30a26f44","familyId":"bmf_f604d3adcb02","name":"ClueAegis-Bench","oneLine":"A benchmark decomposing synthetic image detection into annotated forensic cognitive skills, used to evaluate a proposed detection framework.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-05-24","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.25009","pdf":"https://arxiv.org/pdf/2605.25009","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.25009"},"evidence":{"snippet":"To support this paradigm, we introduce \\textbf{ClueAegis-Bench}, which decomposes synthetic image detection into explicitly annotated forensic cognitive skills for structured evaluation beyond binary classification.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.25009"},"ranking":{},"description":"A benchmark decomposing synthetic image detection into annotated forensic cognitive skills, used to evaluate a proposed detection framework.","whyItMatters":"Supports evidence-based synthetic image detection beyond binary classification.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e1a9944fc2dbaee73847881d165a1755c29f70fe5f78b0452841e4d961ddc873"},"motivation":"The rapid advancement of generative models has made synthetic images increasingly realistic, challenging reliable detection.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.25009","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_ts-skill_e0ee25de","familyId":"bmf_214b8b173f4a","name":"TS-Skill","oneLine":"A controlled benchmark for evaluating analytical skills in time-series question answering: temporal scale selection, temporal localization, and cross-interval integration.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24703","pdf":"https://arxiv.org/pdf/2605.24703","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24703"},"evidence":{"snippet":"We introduce TS-Skill, a controlled benchmark for evaluating three composable analytical skills in TSQA: temporal scale selection (SK1), temporal localization (SK2), and cross-interval integration (SK3).","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24703"},"ranking":{},"description":"A controlled benchmark for evaluating analytical skills in time-series question answering: temporal scale selection, temporal localization, and cross-interval integration.","whyItMatters":"Skill-level evaluation reveals temporal reasoning failures obscured by aggregate scores.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c1aea376bad770f9fb95d59c151329c5270e00ac8a067e369194e3681e82b445"},"motivation":"Large language models (LLMs) and time-series language models (TSLMs) are increasingly applied to time-series question answering (TSQA).","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24703","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_tc-bench_6009a2aa","familyId":"bmf_4b558e2721e0","name":"TC-Bench","oneLine":"A benchmark dataset for tropical cyclone research with an automated construction pipeline, used to probe scientific alignment of vision foundation models.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24782","pdf":"https://arxiv.org/pdf/2605.24782","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24782"},"evidence":{"snippet":"To operationalize this framework, we release TC-Bench, a global, reproducible benchmark dataset with an automated construction pipeline for tropical cyclone research, and show that current VFMs rely on visual shortcuts that collapse in intense regimes, indicating that scientific alignment does not arise as a natural byproduct of scaling alone.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24782"},"ranking":{},"description":"A benchmark dataset for tropical cyclone research with an automated construction pipeline, used to probe scientific alignment of vision foundation models.","whyItMatters":"Examines whether models rely on visual shortcuts rather than structural invariants in scientific domains.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"488472c69ff7d173ad64263cf6b3f92eae30f62ad5092da7cfa42c4932096a96"},"motivation":"While Vision Foundation Models (VFMs) excel at predictive tasks on satellite imagery, their performance can arise from visual correlations rather than underlying structural invariants, making even perception-based out-of-distribution accuracy a poor proxy for scientific utility.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026","evidence":"Accepted at ICML 2026","evidenceUrl":"https://arxiv.org/abs/2605.24782","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026","reviewStatus":"accepted","decisionRaw":"Accepted at ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2605.24782","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted at ICML 2026","level":"author-claim"}]}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_sdr-bench_b95d0afe","familyId":"bmf_78c11886d9c2","name":"SDR-Bench","oneLine":"SDR-Bench evaluates LLM personalization in a two-party framework. It includes 6,279 customer success stories across 22 industries and ~200 enterprises, with a temporally constrained simulation to prevent data leakage. Scoring compares model-generated outreach against human outcomes.","area":"Language & Knowledge","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Logistics"],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2607.20471","pdf":"https://arxiv.org/pdf/2607.20471","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2607.20471"},"evidence":{"snippet":"We release SDR-Bench, a public corpus of 6,279 customer success stories spanning 22 industries and approximately 200 enterprises, served through a temporally constrained simulation that prevents future-data leakage.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2607.20471"},"ranking":{},"description":"SDR-Bench evaluates LLM personalization in a two-party framework. It includes 6,279 customer success stories across 22 industries and ~200 enterprises, with a temporally constrained simulation to prevent data leakage. Scoring compares model-generated outreach against human outcomes.","whyItMatters":"SDR-Bench addresses the gap in evaluating LLM personalization as a two-party problem, where generated messages must induce action in a third party. It provides a reproducible public benchmark for comparing model performance on this task, with validation against field deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c6b19bbe38a867ea12575e93ca406cc3e43b9e476b627ffac8d446f4a6baa118"},"motivation":"Personalization, the act of varying a message to induce action from a specific receiver while keeping sender, channel, and time fixed, has a long tradition in psychology and marketing as a two-party problem in which sender and receiver have independent objectives.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2607.20471","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_mr-lidar_0b6f3704","familyId":"bmf_81c065acf37e","name":"MR-LiDAR","oneLine":"MR-LiDAR is a multi-resolution LiDAR benchmark for roadside perception. It includes point cloud data and annotations from 16-, 32-, 80-, and 128-beam LiDARs in identical scenarios, covering vehicles and vulnerable road users at various distances.","area":"Robotics & Embodied AI","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.RO"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24777","pdf":"https://arxiv.org/pdf/2605.24777","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24777"},"evidence":{"snippet":"To address this gap, we present MR-LiDAR, a controlled multi-resolution LiDAR benchmark for roadside perception diagnostics.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24777"},"ranking":{},"description":"MR-LiDAR is a multi-resolution LiDAR benchmark for roadside perception. It includes point cloud data and annotations from 16-, 32-, 80-, and 128-beam LiDARs in identical scenarios, covering vehicles and vulnerable road users at various distances.","whyItMatters":"The benchmark targets the need for empirical comparison of LiDAR configurations to guide sensor selection in roadside perception systems, addressing cost and performance trade-offs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"1f592aff16e110d7eea06f5cc9a12bc913daddb20cf5dd497fc1356f88a096bf"},"motivation":"LiDAR model selection is a critical issue in roadside sensing systems, as it directly determines both perception capability and deployment cost.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24777","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"general"},{"id":"bm_llmtabbench_435990cf","familyId":"bmf_1ca641025f84","name":"LLMTabBench","oneLine":"LLMTabBench evaluates LLMs on binary tabular classification in zero- and few-shot settings, using real-world and controlled synthetic datasets to study the effect of task descriptions and examples on performance.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.LG"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24417","pdf":"https://arxiv.org/pdf/2605.24417","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24417"},"evidence":{"snippet":"We introduce LLMTabBench, a benchmark for evaluating LLMs on tabular classification under low-data conditions.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24417"},"ranking":{},"description":"LLMTabBench evaluates LLMs on binary tabular classification in zero- and few-shot settings, using real-world and controlled synthetic datasets to study the effect of task descriptions and examples on performance.","whyItMatters":"The benchmark addresses the gap in understanding how LLMs perform on tabular data under low-data regimes, which can inform their deployment in data-scarce applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"aaa39281bc980dd951a3b446a8ee3ba90e342a47e43bbe9a092d65609af20f76"},"motivation":"Supervised classification on tabular data remains a central machine learning task, but its dependence on large labeled datasets limits its applicability in data-scarce settings.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24417","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_himed_f6226988","familyId":"bmf_3f1df78e7570","name":"HiMed","oneLine":"HiMed is a Hindi medical dataset and benchmark suite covering Western and Indian medicine. It evaluates medical reasoning in Hindi across multiple tasks, with data and evaluation scripts publicly released on GitHub.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24635","pdf":"https://arxiv.org/pdf/2605.24635","project":null,"code":"https://github.com/FreedomIntelligence/HiMed","data":null,"hfPaper":"https://huggingface.co/papers/2605.24635"},"evidence":{"snippet":"To this end, we introduce HiMed, a Hindi reasoning medical corpus and benchmark suite covering both Western and Indian medicine.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24635"},"ranking":{},"description":"HiMed is a Hindi medical dataset and benchmark suite covering Western and Indian medicine. It evaluates medical reasoning in Hindi across multiple tasks, with data and evaluation scripts publicly released on GitHub.","whyItMatters":"Hindi remains underrepresented in medical LLMs; this benchmark enables systematic evaluation of cross-lingual medical reasoning, helping reduce performance gaps between English and Hindi in healthcare applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a880022ffd5b04467b15cdcddb91166da04621c5be64ce934bd883033151aacc"},"motivation":"Medical large language models hold promise for reducing healthcare disparities, yet Hindi remains severely underrepresented.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24635","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"FreedomIntelligence","organizationType":"academic-lab","sourceUrl":"https://github.com/FreedomIntelligence/HiMed","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_globaldentbench_f42d9d35","familyId":"bmf_6790f75fc2f9","name":"GlobalDentBench","oneLine":"GlobalDentBench evaluates LLM clinical reasoning in dentistry with 8,978 expert-validated questions across 14 specialties and 88 countries, covering multiple-choice, short-answer, and case-based formats at three reasoning levels.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.24636","pdf":"https://arxiv.org/pdf/2605.24636","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24636"},"evidence":{"snippet":"Here we introduce GlobalDentBench, the first multinational dental benchmark, featuring a taxonomy that encompasses 14 dental specialties across 88 countries and regions spanning six continents.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24636"},"ranking":{},"description":"GlobalDentBench evaluates LLM clinical reasoning in dentistry with 8,978 expert-validated questions across 14 specialties and 88 countries, covering multiple-choice, short-answer, and case-based formats at three reasoning levels.","whyItMatters":"It provides a multinational dental benchmark with expert calibration to assess knowledge recall, routine and individualized reasoning, revealing safety risks in LLM clinical recommendations and supporting rigorous validation before deployment.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b3bfc579c8c91cd9b50169e0a1580df569385a8933a4ee21a87069f1b071b5a2"},"motivation":"While large language models (LLMs) hold transformative potential for medicine, their reasoning robustness and safety in real-world clinical scenarios remain critically underexplored, particularly in dentistry.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24636","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GlobalDentBench Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2605.24636","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_foodmonitor_339fb661","familyId":"bmf_bf993ac0f8c6","name":"FoodMonitor","oneLine":"Evaluates multimodal large language models on explainable compliance analysis in commercial kitchen surveillance, using video clips with dual-channel annotations and a composite metric.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24503","pdf":"https://arxiv.org/pdf/2605.24503","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24503"},"evidence":{"snippet":"We introduce FoodMonitor, a benchmark for explainable compliance analysis in commercial kitchen surveillance.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24503"},"ranking":{},"description":"Evaluates multimodal large language models on explainable compliance analysis in commercial kitchen surveillance, using video clips with dual-channel annotations and a composite metric.","whyItMatters":"Provides a protocol for assessing both spatial localization and semantic rule understanding in video anomaly detection, addressing a gap in existing event-level binary classification benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f0e97ddd21d51305ecaccbde1816523e789e40ab0cfb933f1004921b1d6f56c0"},"motivation":"As AI-powered compliance monitoring becomes increasingly important in public governance and industrial safety, the ability to provide verifiable evidence and traceable accountability signals is essential.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24503","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_egoprox_628390c5","familyId":"bmf_c4115f8c172a","name":"EgoProx","oneLine":"EgoProx evaluates multimodal large language models on egocentric 3D proximity reasoning, with tasks organized along a cognitive hierarchy covering intention, exploration, exploitation, and chain-of-actions reasoning.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning","Geometric reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24456","pdf":"https://arxiv.org/pdf/2605.24456","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24456"},"evidence":{"snippet":"To this end, we introduce EgoProx, a benchmark for egocentric 3D proximity reasoning.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24456"},"ranking":{},"description":"EgoProx evaluates multimodal large language models on egocentric 3D proximity reasoning, with tasks organized along a cognitive hierarchy covering intention, exploration, exploitation, and chain-of-actions reasoning.","whyItMatters":"It addresses the gap in assessing embodied 3D spatial reasoning in MLLMs, a capability important for robotics and augmented reality applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0dd961362af71913f44d54eaf7cad948ed6e418826fcf3159123ab8a92688ad7"},"motivation":"Humans constantly reason about 3D proximity, the relations between their body and surrounding objects, to guide perception and action in daily life.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"CVPR 2026","evidence":"Accepted to CVPR 2026","evidenceUrl":"https://arxiv.org/abs/2605.24456","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"CVPR 2026","reviewStatus":"accepted","decisionRaw":"Accepted to CVPR 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2605.24456","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted to CVPR 2026","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_cybermaskqa_2283c85d","familyId":"bmf_5ce23c89d555","name":"CyberMaskQA","oneLine":"The evaluation object is a dataset for privacy-aware cybersecurity question answering, covering key security domains with private entity labels. The main capability evaluated is QA accuracy and masking performance.","area":"Language & Knowledge","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CR"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24765","pdf":"https://arxiv.org/pdf/2605.24765","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24765"},"evidence":{"snippet":"To address this gap, we introduce CYBERMASKQA, a privacy-aware QA benchmark covering key security domains.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24765"},"ranking":{},"description":"The evaluation object is a dataset for privacy-aware cybersecurity question answering, covering key security domains with private entity labels. The main capability evaluated is QA accuracy and masking performance.","whyItMatters":"The evaluation gap is the lack of context-rich datasets for privacy-preserving QA in cybersecurity, which hinders progress. This benchmark's value is in enabling controlled information disclosure and studying privacy-utility trade-offs for deployable models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"62a9dc9cf905a20e3ecbddde28efeab2638bc9ec43694cb114944b77c4e58a21"},"motivation":"Large language models (LLMs) are increasingly applied to cybersecurity question answering (QA) for critical tasks such as incident response and vulnerability analysis.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24765","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_cod10k-c_fc1b1fcc","familyId":"bmf_3d7b81bb10c7","name":"COD10K-C","oneLine":"COD10K-C evaluates camouflaged object detection models under 8 corruption types at 5 severity levels, yielding 40 conditions and 81,040 image pairs based on COD10K. Models are scored on Dice and other standard metrics for segmentation robustness.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Robustness"],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2606.02603","pdf":"https://arxiv.org/pdf/2606.02603","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2606.02603"},"evidence":{"snippet":"We present COD10K-C, a corruption robustness benchmark based on COD10K.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2606.02603"},"ranking":{},"description":"COD10K-C evaluates camouflaged object detection models under 8 corruption types at 5 severity levels, yielding 40 conditions and 81,040 image pairs based on COD10K. Models are scored on Dice and other standard metrics for segmentation robustness.","whyItMatters":"Standard camouflaged object detection benchmarks measure performance on clean images only, while real-world captures include blur, noise, weather, and compression artifacts. This benchmark quantifies robustness drops under such corruptions, enabling selection of models that degrade gracefully in practical conditions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"ad4f73f8cfaff6e64c8aff53929fe65e5a172c6bd16ce20d29fd954fe6999de2"},"motivation":"Camouflaged object detection has improved substantially, but most standard benchmarks evaluate models only on clean images.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2606.02603","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness","Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_chaosbench-logic_c5cd44be","familyId":"bmf_173654d46d35","name":"ChaosBench-Logic","oneLine":"ChaosBench-Logic v2 evaluates LLM logical reasoning over dynamical systems with 40,886 questions across 165 systems, 27 first-order logic predicates, and 78 axiom edges. The CARE protocol measures calibration and adversarial robustness, reporting metrics such as MCC.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24305","pdf":"https://arxiv.org/pdf/2605.24305","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24305"},"evidence":{"snippet":"We present ChaosBench-Logic v2, a 40,886-question benchmark over 165 dynamical systems with 27 FOL predicates and 78 axiom edges, together with CARE (Calibration- and Adversarial-Robust Evaluation), a protocol that surfaces these pathologies.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24305"},"ranking":{},"description":"ChaosBench-Logic v2 evaluates LLM logical reasoning over dynamical systems with 40,886 questions across 165 systems, 27 first-order logic predicates, and 78 axiom edges. The CARE protocol measures calibration and adversarial robustness, reporting metrics such as MCC.","whyItMatters":"Binary accuracy on reasoning benchmarks hides failures like prior collapse and inconsistency under paraphrase. This benchmark surfaces these pathologies and quantifies reasoning about parameter-dependent dynamics, guiding improvements in LLM reasoning capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9c846a7cbecc884598e92cb84a3b54515adcf6bc3b71d939178af3da7363ec4f"},"motivation":"Standard accuracy on binary reasoning benchmarks hides critical failure modes: prior collapse, inconsistency under paraphrase, and inability to reason about parameter-dependent dynamics.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24305","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_avbench_ce8c8830","familyId":"bmf_edaffec7807d","name":"AVBench","oneLine":"AVBench evaluates audio-video generative models across ten fine-grained dimensions covering visual quality, audio quality, and cross-modal consistency for human-centric scenarios. It uses specialized evaluators trained via preference learning and provides continuous scores from prediction confidence.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.24652","pdf":"https://arxiv.org/pdf/2605.24652","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24652"},"evidence":{"snippet":"To address these issues, we introduce AVBench, a fully automated benchmark tailored for human-centric AV generation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24652"},"ranking":{},"description":"AVBench evaluates audio-video generative models across ten fine-grained dimensions covering visual quality, audio quality, and cross-modal consistency for human-centric scenarios. It uses specialized evaluators trained via preference learning and provides continuous scores from prediction confidence.","whyItMatters":"Existing benchmarks for AV generation are coarse and rely on generic multimodal LLMs, leading to inaccurate assessments. AVBench offers automated, human-aligned evaluation with fine-grained metrics, enabling reliable model comparison and serving as a potential reward signal for RLHF.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c2a081dc2cfddf81dcb28c0d41671c9feedc190ff6f47141701fe425db4215e7"},"motivation":"Rapid advances in audio-video (AV) generation have enabled high-fidelity synthesis with synchronized sound, particularly for human-related scenarios involving speech and interactions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24652","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_astromind_587e5d1f","familyId":"bmf_f3195093bff0","name":"AstroMind","oneLine":"AstroMind evaluates LLM reasoning about spacecraft behavior across intent inference, maneuver parameter estimation, and threat assessment, using physics-grounded simulations with realistic sensing noise. Metrics capture semantic correctness and quantitative consistency under physical constraints.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-23","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.24573","pdf":"https://arxiv.org/pdf/2605.24573","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24573"},"evidence":{"snippet":"AstroMind is a physics-grounded benchmark designed to close that gap.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24573"},"ranking":{},"description":"AstroMind evaluates LLM reasoning about spacecraft behavior across intent inference, maneuver parameter estimation, and threat assessment, using physics-grounded simulations with realistic sensing noise. Metrics capture semantic correctness and quantitative consistency under physical constraints.","whyItMatters":"Space situational awareness lacks benchmarks for reasoning about why spacecraft maneuver, not just detection. AstroMind provides a shared test that combines physics accuracy and tactical interpretation, enabling comparison of models on this critical reasoning task.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"f248c443fec22a71ed76885eb2b752d4b2739d32e164f4b1dbde09b60c6c2669"},"motivation":"Understanding why a spacecraft maneuvers -- rather than simply that it did -- is an increasingly important problem for space domain awareness as Earth orbits grow crowded and contested.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24573","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_stereogenbench_78fe1ed3","familyId":"bmf_a246b7b4c19d","name":"StereoGenBench","oneLine":"StereoGenBench is a synthetic multi-camera benchmark for stereo generation, rendered in Unreal Engine with a rigid six-camera array. It provides calibrated view pairs across baseline regimes, along with RGB, metric depth, intrinsics, and poses for each scene.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Inspectable","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.23237","pdf":"https://arxiv.org/pdf/2605.23237","project":null,"code":null,"data":"https://huggingface.co/datasets/stereo-dataset/stereo-dataset","hfPaper":"https://huggingface.co/papers/2605.23237"},"evidence":{"snippet":"We introduce StereoGenBench, a synthetic Unreal Engine benchmark designed to make baseline-regime sensitivity and target-camera consistency measurable under matched scene content.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":1604,"hfDatasetLikes":2},"source":{"type":"arxiv","id":"2605.23237"},"ranking":{},"description":"StereoGenBench is a synthetic multi-camera benchmark for stereo generation, rendered in Unreal Engine with a rigid six-camera array. It provides calibrated view pairs across baseline regimes, along with RGB, metric depth, intrinsics, and poses for each scene.","whyItMatters":"Stereo generation and view synthesis require controlled baseline and intrinsics for evaluation, which existing resources lack. This benchmark enables measuring sensitivity to baseline regimes and target-camera consistency under matched scene content, supporting development of stereo generation models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b61dfbd5cb24164ed4560890f3148878bf3ab598d1ae9ce73b14f8edaa79b960"},"motivation":"Stereo image and video generation, stereo geometry estimation, and condition-controlled view synthesis require paired data in which the variables that determine binocular geometry -- camera baseline, intrinsics, scene depth, and camera motion -- are known and controllable.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.23237","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_skillevolbench_997ef790","familyId":"bmf_0d32e638a5b1","name":"SkillEvolBench","oneLine":"SkillEvolBench evaluates whether LLM agents can distill episodic experience into reusable procedural skills. It includes 180 tasks across six environments, with acquisition tasks and frozen deployment tasks testing context shift, adversarial shortcuts, and composition. Scoring uses success rates and other agent metrics.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24117","pdf":"https://arxiv.org/pdf/2605.24117","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24117"},"evidence":{"snippet":"We introduce SkillEvolBench, a diagnostic benchmark for evaluating this step from experience reuse to skill formation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":22,"hfDailySubmittedAt":"2026-05-26T00:00:00.000Z","githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24117"},"ranking":{},"description":"SkillEvolBench evaluates whether LLM agents can distill episodic experience into reusable procedural skills. It includes 180 tasks across six environments, with acquisition tasks and frozen deployment tasks testing context shift, adversarial shortcuts, and composition. Scoring uses success rates and other agent metrics.","whyItMatters":"It is unclear whether LLM agents can form durable procedural skills from experience. SkillEvolBench provides a diagnostic testbed comparing skill-based learning against raw-trajectory reuse, informing agent design and training.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"67b22643bad860b49d7efe0353d87e1c99be64e10066d727165156af7747dd3c"},"motivation":"Large language model (LLM) agents accumulate rich episodic trajectories while solving real-world tasks, but it remains unclear whether such experience can be distilled into reusable procedural skills.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24117","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_poisonforge_b7c26795","familyId":"bmf_d4f13d62e72d","name":"PoisonForge","oneLine":"PoisonForge benchmarks task-level targeted poisoning of instruction-tuned LLMs, parameterizing bias type, poisoning mode, appearance count, and target output length. It evaluates 12 open-weight models across five families with primarily 1% poison budget, using attack success rate as the main metric.","area":"Safety & Trustworthiness","applicationDomains":["Cybersecurity","Transport & Logistics"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity","Logistics"],"capabilities":[],"topics":["cs.CR"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.23168","pdf":"https://arxiv.org/pdf/2605.23168","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.23168"},"evidence":{"snippet":"We introduce PoisonForge, a benchmark that parameterizes this threat along four dimensions (bias type, poisoning mode, appearance count, and target output length) and evaluates 12 open-weight models (from 2B to 32B parameters) across five families under a primarily 1% poison budget.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.23168"},"ranking":{},"description":"PoisonForge benchmarks task-level targeted poisoning of instruction-tuned LLMs, parameterizing bias type, poisoning mode, appearance count, and target output length. It evaluates 12 open-weight models across five families with primarily 1% poison budget, using attack success rate as the main metric.","whyItMatters":"Data supply chain poisoning poses a real threat when fine-tuning on unvetted data. PoisonForge quantifies vulnerability across models and configurations, highlighting that design choices rather than scale drive risk, aiding in risk assessment and mitigation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0b816d0d5409e4d3ebe05d45294a5e88bfd8eff3e42dcbb48db39ca9b9738f72"},"motivation":"When practitioners fine-tune LLMs on unvetted datasets, an adversary can exploit the data supply chain through task-level poisoning: inserting a small number of crafted instruction-response pairs that cause the model to embed attacker-specified entities, such as a country, in outputs for a targeted task family while behaving normally elsewhere.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.23168","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"cross-domain"},{"id":"bm_nqp-bench_0c5c1e38","familyId":"bmf_164c8f5b65ff","name":"NQP-Bench","oneLine":"NQP-Bench is a dataset within the OnePred paper for evaluating next-query prediction in multi-turn conversations. It spans three diverse subsets and is used to compare OnePred against baselines.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.23668","pdf":"https://arxiv.org/pdf/2605.23668","project":null,"code":"https://github.com/ZBWpro/OnePred","data":null,"hfPaper":"https://huggingface.co/papers/2605.23668"},"evidence":{"snippet":"To establish a rigorous testbed, we introduce NQP-Bench, spanning three diverse subsets.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.23668"},"ranking":{},"description":"NQP-Bench is a dataset within the OnePred paper for evaluating next-query prediction in multi-turn conversations. It spans three diverse subsets and is used to compare OnePred against baselines.","whyItMatters":"Next-query prediction lacks dedicated benchmarks; NQP-Bench provides a testbed for this task, enabling evaluation of proactive conversational systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"091f6a8b6836370140597e7d8a58fcbdf76e912f03af5ce6514329170c654c54"},"motivation":"Although large language model (LLM) conversational systems process millions of multi-turn dialogues daily, they remain fundamentally reactive: they respond only after the user types a query.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.23668","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_jedi_9de96442","familyId":"bmf_17779b11c2c9","name":"JEDI","oneLine":"JEDI is a benchmark suite for the Java Stream API, generated by converting SQL benchmarks into Java benchmarks. It includes both stream-based and imperative query implementations to evaluate performance of parallelization strategies.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["cs.PL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.23543","pdf":"https://arxiv.org/pdf/2605.23543","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.23543"},"evidence":{"snippet":"In this work we present JEDI, a benchmark suite that targets the Stream API.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.23543"},"ranking":{},"description":"JEDI is a benchmark suite for the Java Stream API, generated by converting SQL benchmarks into Java benchmarks. It includes both stream-based and imperative query implementations to evaluate performance of parallelization strategies.","whyItMatters":"There is a lack of benchmarks for the Java Stream API, making it difficult to optimize the API and analyze stream performance. JEDI provides a dedicated suite to guide developers in writing efficient code and researchers in optimizing the API.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a5d2732a73a0ca31ba9936ca344cff1a640d97dba4466c0f908814fb31df80b5"},"motivation":"The Java Stream API aims at increasing developer productivity thanks to an easy-to-read declarative syntax to express computations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"publication_reported","venue":"2026 IEEE/ACM 48th International Conference on Software Engineering (ICSE '26)","evidence":"2026 IEEE/ACM 48th International Conference on Software Engineering (ICSE '26)","evidenceUrl":"https://arxiv.org/abs/2605.23543","source":"arxiv-journal-reference","evidenceLevel":"strong-author-metadata","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publications":[{"venueName":"2026 IEEE/ACM 48th International Conference on Software Engineering (ICSE '26)","publicationStatus":"published","evidence":[{"sourceType":"arxiv-journal-reference","sourceUrl":"https://arxiv.org/abs/2605.23543","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"2026 IEEE/ACM 48th International Conference on Software Engineering (ICSE '26)","level":"strong-author-metadata"}]}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_intentionnav_63acc227","familyId":"bmf_5dd62d05b1f1","name":"IntentionNav","oneLine":"IntentionNav evaluates active object search from implicit human instructions in 176 Isaac Sim scenes. Episodes provide free-text intent, RGB-D observations, and pose, with the target object withheld. The benchmark includes 500 intents over 64 categories and four intent modes.","area":"Robotics & Embodied AI","applicationDomains":["Robotics & Autonomous Systems"],"primaryDomain":"Robotics & Autonomous Systems","industrySectors":["Robotics"],"capabilities":[],"topics":["cs.CV"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.23187","pdf":"https://arxiv.org/pdf/2605.23187","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.23187"},"evidence":{"snippet":"We study this setting as intent-driven object navigation and introduce IntentionNav, a diagnostic benchmark for active object search from implicit human instructions.","reasonCodes":["exact coined title identity tied to benchmark evidence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.23187"},"ranking":{},"description":"IntentionNav evaluates active object search from implicit human instructions in 176 Isaac Sim scenes. Episodes provide free-text intent, RGB-D observations, and pose, with the target object withheld. The benchmark includes 500 intents over 64 categories and four intent modes.","whyItMatters":"Addresses the gap in object navigation where agents must infer targets from indirect human intent, showing persistent bottlenecks in target selection and terminal localization.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5b88d5dd64953bce8f1aa1d7a5e40bce0026236812ec80b12c0b99a1c2e1165c"},"motivation":"Existing object navigation benchmarks usually tell an embodied agent which object category to find, such as microwave or chair.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.23187","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Robotics & Embodied Intelligence"],"domainScope":"specific"},{"id":"bm_fastkernels_60e561f1","familyId":"bmf_c3b50d55e8f5","name":"FastKernels","oneLine":"FastKernels evaluates GPU kernel generation for production inference across 46 representative architectures spanning 8 categories. It measures correctness and speedup of candidate kernels against reference implementations at multiple abstraction levels, from single-kernel to end-to-end serving.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":["Code generation"],"topics":["Code"],"construction":"Original Synthetic","annotation":"Machine Generated","readiness":"Runnable","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.23215","pdf":"https://arxiv.org/pdf/2605.23215","project":null,"code":"https://github.com/Snowflake-AI-Research/fastkernels","data":null,"hfPaper":"https://huggingface.co/papers/2605.23215"},"evidence":{"snippet":"We introduce FastKernels, a kernel benchmark built around a minimal set of 46 representative architectures spanning 8 categories, whose kernels collectively subsume those of 96.2% (409/425) of HuggingFace Transformers architectures.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":8,"hfDailySubmittedAt":"2026-05-27T00:00:00.000Z","githubStars":5,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.23215"},"ranking":{},"description":"FastKernels evaluates GPU kernel generation for production inference across 46 representative architectures spanning 8 categories. It measures correctness and speedup of candidate kernels against reference implementations at multiple abstraction levels, from single-kernel to end-to-end serving.","whyItMatters":"Existing kernel benchmarks reward sandbox optimizations that fail in production. FastKernels aligns evaluation with real inference frameworks, providing a benchmark where scores translate to throughput improvements in production codebases.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c697a4fc536c397d33cd9246d6cabdeef8722d60539728d93d563042db782833"},"motivation":"LLM-based agents for GPU kernel generation are advancing rapidly, yet their progress is fundamentally constrained by the benchmarks they optimize against.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.23215","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Snowflake AI Research","organizationType":"company-research-lab","sourceUrl":"https://github.com/Snowflake-AI-Research/fastkernels","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_evocode-bench_60a115e8","familyId":"bmf_7accae8d1fc6","name":"EvoCode-Bench","oneLine":"EvoCode-Bench evaluates coding agents in multi-turn iterative interactions, with 26 stateful coding tasks and 227 evaluated rounds. Each task preserves the agent's workspace for 5-15 rounds, specifies requirements via observable behavior, and uses cumulative executable tests to verify new and prior requirements. Scoring uses MT@4, a four-attempt fail-stop multi-round score, and SR, a single-round score from a reference-completed prior state.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24110","pdf":"https://arxiv.org/pdf/2605.24110","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24110"},"evidence":{"snippet":"We introduce EvoCode-Bench, a benchmark of 26 stateful coding tasks and 227 evaluated rounds.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24110"},"ranking":{},"description":"EvoCode-Bench evaluates coding agents in multi-turn iterative interactions, with 26 stateful coding tasks and 227 evaluated rounds. Each task preserves the agent's workspace for 5-15 rounds, specifies requirements via observable behavior, and uses cumulative executable tests to verify new and prior requirements. Scoring uses MT@4, a four-attempt fail-stop multi-round score, and SR, a single-round score from a reference-completed prior state.","whyItMatters":"Existing coding benchmarks typically evaluate a single specification and final output, missing the ability to handle evolving requirements. EvoCode-Bench fills this gap by assessing whether agents can maintain a working codebase over multiple rounds. The gap between SR and MT@4 scores reveals that high single-turn performance does not guarantee sustained multi-turn success, providing a more relevant evaluation for real-world iterative development.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6c10356ec27b25e4d29c0d964ed27fd97db09167b3f369ba416cb2b21b39da24"},"motivation":"Coding agents are increasingly used as iterative development partners, but most benchmarks still evaluate one specification followed by one final assessment.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24110","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Harbor","organizationType":"community","sourceUrl":"https://huggingface.co/papers/2605.24110","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_drivespatial_f218db55","familyId":"bmf_f22a23f585f9","name":"DRIVESPATIAL","oneLine":"DriveSpatial is a benchmark of 15.6K human-verified QA pairs across 20 tasks from five AD datasets, evaluating VLMs on cognitive scene construction, multi-view relational understanding, temporal reasoning, and generalization.","area":"Vision & 3D","applicationDomains":["Transport & Logistics"],"primaryDomain":"Transport & Logistics","industrySectors":["Automotive"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.23176","pdf":"https://arxiv.org/pdf/2605.23176","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.23176"},"evidence":{"snippet":"We introduce DriveSpatial, a benchmark of 15.6K human-verified QA pairs across 20 tasks from five large-scale AD datasets.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.23176"},"ranking":{},"description":"DriveSpatial is a benchmark of 15.6K human-verified QA pairs across 20 tasks from five AD datasets, evaluating VLMs on cognitive scene construction, multi-view relational understanding, temporal reasoning, and generalization.","whyItMatters":"Existing AD vision-language benchmarks focus on static, single-view QA, leaving unclear whether VLMs can reason over dynamic driving scenes. DriveSpatial provides a multi-sourced, human-verified protocol to measure spatiotemporal reasoning and reveals a substantial human-model gap.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"0a5d90c49b179b9c945fd98805ee2f9c3e4a362e7f16159a1657981f60a0f746"},"motivation":"Spatiotemporal intelligence in autonomous driving (AD) requires an agent to integrate multi-view observations into a coherent scene representation, maintain object continuity across viewpoints and time, and reason about spatial relations, interactions, and future dynamics.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.23176","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_drinq_46ba6b80","familyId":"bmf_b8e493ef53e0","name":"DRInQ","oneLine":"The evaluation targets conversational implicature in question utterances, using a semi-automated pipeline to generate question-context-interpretation instances with controlled variation.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24267","pdf":"https://arxiv.org/pdf/2605.24267","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24267"},"evidence":{"snippet":"We introduce DRinQ, a benchmark for evaluating pragmatic reasoning about conversational implicature in question utterances, designed to isolate pragmatic variation while holding each question's surface form fixed.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24267"},"ranking":{},"description":"The evaluation targets conversational implicature in question utterances, using a semi-automated pipeline to generate question-context-interpretation instances with controlled variation.","whyItMatters":"This evaluation probes the gap between generation and inference in pragmatic reasoning, but no public comparison path is provided beyond the paper's findings.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"b11a5d2686f1106926a2e82766ebf4db4fb0c795ea44823b5010389f0f204c61"},"motivation":"Human conversation relies heavily on conversational implicature, in which speakers convey meanings that are suggested rather than explicitly stated.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24267","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_ddx-trace_5ca2d5dc","familyId":"bmf_282505fdf4da","name":"DDX-TRACE","oneLine":"DDX-TRACE evaluates medical diagnostic trajectories in multimodal neuroradiology over 211 physician-adjudicated cases, where models request imaging studies, update differential diagnoses, and stop with a localized final diagnosis under hidden evidence.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.23629","pdf":"https://arxiv.org/pdf/2605.23629","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.23629"},"evidence":{"snippet":"We introduce DDX-TRACE, a physician-adjudicated benchmark for multimodal neuroradiology that evaluates diagnostic trajectories under hidden evidence over 211 challenging cases.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.23629"},"ranking":{},"description":"DDX-TRACE evaluates medical diagnostic trajectories in multimodal neuroradiology over 211 physician-adjudicated cases, where models request imaging studies, update differential diagnoses, and stop with a localized final diagnosis under hidden evidence.","whyItMatters":"Traditional medical AI benchmarks reward final answers, overlooking workup quality. DDX-TRACE measures evidence-supported diagnostic processes, providing a more realistic and actionable evaluation for clinical decision support.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"3159448f0f991d33571a2865fb65c3ac34606e6b948e262d3bcd2f99d0aff3ca"},"motivation":"Medical diagnosis is not a single prediction from a fully specified vignette.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.23629","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"DDX-TRACE Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2605.23629","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_cronos_36061fdb","familyId":"bmf_6526b48f897f","name":"CRONOS","oneLine":"CRONOS is an intervention-based benchmark for evaluating counterfactual physical consistency in video prediction models. It provides a photorealistic Unreal Engine environment with controlled videos of physical events (collision, occlusion, fall) while intervening on viewpoint, scene, object category, and object appearance. The evaluation protocol defines six metrics computed via video segmentation, tracking, 3D reconstruction, and VLM-based task performance.","area":"Vision & 3D","applicationDomains":["Cybersecurity"],"primaryDomain":"Cybersecurity","industrySectors":["Cybersecurity"],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.23699","pdf":"https://arxiv.org/pdf/2605.23699","project":null,"code":"https://github.com/GenIntel/CRONOS-benchmark","data":null,"hfPaper":"https://huggingface.co/papers/2605.23699"},"evidence":{"snippet":"We introduce CRONOS, an intervention-based benchmark designed to evaluate counterfactual physical consistency: whether a model's predictions of physical events respond appropriately to controlled changes in the visual input, such as variations of scene context, viewpoint, object appearance, and object category.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":11,"hfDailySubmittedAt":"2026-05-26T00:00:00.000Z","githubStars":8,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.23699"},"ranking":{},"description":"CRONOS is an intervention-based benchmark for evaluating counterfactual physical consistency in video prediction models. It provides a photorealistic Unreal Engine environment with controlled videos of physical events (collision, occlusion, fall) while intervening on viewpoint, scene, object category, and object appearance. The evaluation protocol defines six metrics computed via video segmentation, tracking, 3D reconstruction, and VLM-based task performance.","whyItMatters":"Current video models often rely on superficial correlations rather than causal structure. CRONOS enables systematic diagnosis of how prediction quality degrades under controlled interventions, providing a concrete target for developing models robust to variations in viewpoint and context.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c0521a32387001d3b42c041ed4cfef544c0ae34c4b83ee919a0d26e527159a47"},"motivation":"Video prediction is increasingly viewed as a path toward generalizable world models, yet it remains unclear whether these systems learn underlying causal structure or merely exploit superficial visual correlations for future prediction.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.23699","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"GenIntel","organizationType":"academic-lab","sourceUrl":"https://github.com/GenIntel/CRONOS-benchmark","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_contextecho_fd140a2e","familyId":"bmf_2ceaf1c7c690","name":"ContextEcho","oneLine":"ContextEcho measures persona drift in long agentic-coding sessions, combining a 25-probe identity suite, snapshot-then-probe protocol, and three anonymized Claude Code sessions.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24279","pdf":"https://arxiv.org/pdf/2605.24279","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24279"},"evidence":{"snippet":"We introduce ContextEcho, a benchmark and reusable harness for measuring persona drift at deployment scale.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24279"},"ranking":{},"description":"ContextEcho measures persona drift in long agentic-coding sessions, combining a 25-probe identity suite, snapshot-then-probe protocol, and three anonymized Claude Code sessions.","whyItMatters":"Persona drift can affect user trust and model reliability in real-world deployment, but existing evaluations may miss it. ContextEcho provides a framework for auditing persona consistency across long sessions.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"a75d6c933a3fd86bd10ac632ca7cf3fd7d7d292c79a2627f23d661968dbb2a42"},"motivation":"A frontier language model's acknowledged \"helpful programming assistant\" persona does not survive long agentic-coding sessions in the deployment regime that production products actually run.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24279","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cast-bench_2c1a3e97","familyId":"bmf_ff0b4c73322f","name":"CaST-Bench","oneLine":"CaST-Bench evaluates causal chain-grounded spatio-temporal reasoning in video question answering. It contains 2,066 questions over 1,015 videos with causal chains annotated as temporal segments and bounding-box tracks. The evaluation suite includes metrics for answer correctness and visual evidence grounding.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":0.95,"links":{"report":"https://arxiv.org/abs/2605.23216","pdf":"https://arxiv.org/pdf/2605.23216","project":"https://woven-by-toyota.github.io/CaST-Bench","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.23216"},"evidence":{"snippet":"To address this gap, we introduce CaST-Bench, a benchmark for Causal Chain-Grounded Spatio-Temporal Video Reasoning.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.23216"},"ranking":{},"description":"CaST-Bench evaluates causal chain-grounded spatio-temporal reasoning in video question answering. It contains 2,066 questions over 1,015 videos with causal chains annotated as temporal segments and bounding-box tracks. The evaluation suite includes metrics for answer correctness and visual evidence grounding.","whyItMatters":"Existing video QA benchmarks lack fine-grained causal grounding. CaST-Bench provides a protocol to assess whether VLMs can identify and localize causal evidence chains, supporting progress in transparent and reliable video understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"187f0b5a364b82dcc95b805fe66deb2e12a25edae94d121a6f5d50120adedf25"},"motivation":"Cause-and-effect reasoning in video is a significant challenge for Vision-Language Models (VLMs), as it requires going beyond surface-level perception to a deeper understanding of causal mechanisms.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.23216","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Woven by Toyota","organizationType":"company-research-lab","sourceUrl":"https://woven-by-toyota.github.io/CaST-Bench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_avalanchebench_c5aa4b06","familyId":"bmf_970926f050ea","name":"AvalancheBench","oneLine":"AvalancheBench evaluates enterprise data agents on latent world recovery, scoring analytical understanding of segments, drivers, temporal events, and relationships from generated observations.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.DB"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-22","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.24183","pdf":"https://arxiv.org/pdf/2605.24183","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24183"},"evidence":{"snippet":"We introduce AvalancheBench, a benchmark for evaluating enterprise data agents through \\emph{latent world recovery}.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24183"},"ranking":{},"description":"AvalancheBench evaluates enterprise data agents on latent world recovery, scoring analytical understanding of segments, drivers, temporal events, and relationships from generated observations.","whyItMatters":"The evaluation gap is the need for controlled diagnostics of whether agents recover analytical structure behind enterprise data, with a rubric for partial credit and error propagation analysis.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"5bf9af3041eb34538b42c92e5c23e6bd35f14f5971bc4b3454c667256e0db942"},"motivation":"We introduce AvalancheBench, a benchmark for evaluating enterprise data agents through \\emph{latent world recovery}.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24183","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_videoodyssey_004b7012","familyId":"bmf_62f3a2834934","name":"VideoOdyssey","oneLine":"VideoOdyssey evaluates models on ultra-long-context video understanding using videos averaging 109 minutes across 11 domains, with two subsets for visual and audio-visual understanding. Tasks include question answering with continuous certificates averaging 16 and 12.8 minutes respectively.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Long Context"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22907","pdf":"https://arxiv.org/pdf/2605.22907","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.22907"},"evidence":{"snippet":"Driven by this metric, we introduce VideoOdyssey, a benchmark specifically designed for ultra-long-context and omni-modal video understanding.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":4,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.22907"},"ranking":{},"description":"VideoOdyssey evaluates models on ultra-long-context video understanding using videos averaging 109 minutes across 11 domains, with two subsets for visual and audio-visual understanding. Tasks include question answering with continuous certificates averaging 16 and 12.8 minutes respectively.","whyItMatters":"Existing long-video benchmarks often only test short segments, failing to capture the cognitive load of continuous reasoning over long spans. VideoOdyssey's multi-level continuous certificates provide a diagnostic for evaluating model performance across varying context lengths, addressing a gap in measuring true long-context and omni-modal understanding.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"97f514bf9efdb638edb1da47ed2b9d10b6559c39c6935a43de73a20e82a9d00a"},"motivation":"Real-world long video understanding requires models to perform continuous tracking, information integration and memory retention over massive temporal spans within extreme video durations.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.22907","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception","Long Context & Memory"],"domainScope":"general"},{"id":"bm_vgenst-bench_f8fb9973","familyId":"bmf_3236a25f97a3","name":"VGenST-Bench","oneLine":"VGenST-Bench evaluates spatio-temporal reasoning in multimodal large language models using 1,200 procedurally generated videos with controlled scene composition, camera trajectory, and reasoning targets. It covers 12 reasoning tasks across three spatial scales and 12 QA types across three reasoning levels, with multiple choice and open-ended question variants.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22570","pdf":"https://arxiv.org/pdf/2605.22570","project":"https://zinosii.github.io/VGenST-Bench/","code":"https://github.com/zinosii/VGenST-Bench","data":null,"hfPaper":"https://huggingface.co/papers/2605.22570"},"evidence":{"snippet":"In this paper, we introduce VGenST-Bench, a video benchmark that employs generative models to actively synthesize highly controlled and diverse evaluation scenarios.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":24,"hfDailySubmittedAt":"2026-05-25T00:00:00.000Z","githubStars":12,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.22570"},"ranking":{},"description":"VGenST-Bench evaluates spatio-temporal reasoning in multimodal large language models using 1,200 procedurally generated videos with controlled scene composition, camera trajectory, and reasoning targets. It covers 12 reasoning tasks across three spatial scales and 12 QA types across three reasoning levels, with multiple choice and open-ended question variants.","whyItMatters":"Existing video reasoning benchmarks rely on static or passively curated content, limiting fine-grained diagnosis. VGenST-Bench uses active synthesis to enable controlled and diverse evaluation of fine-grained spatio-temporal reasoning, supporting model comparison and targeted improvement in MLLMs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"da5d35794b1ed6f3b0129bc5f428a0f94f85a866a439b03f27a0decc4c7653cf"},"motivation":"Spatio-temporal reasoning is a core capability for Multimodal Large Language Models (MLLMs) operating in the real world.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.22570","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Sungkyunkwan University","organizationType":"academic-lab","sourceUrl":"https://zinosii.github.io/VGenST-Bench/","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_swe-mutation_d27a11bf","familyId":"bmf_a72ddf5211a2","name":"SWE-Mutation","oneLine":"SWE-Mutation evaluates LLM-generated test suites in software engineering by using 2,636 mutated variants derived from 800 original instances across nine programming languages, measuring verification and detection rates.","area":"Code & Software","applicationDomains":[],"primaryDomain":"General AI","industrySectors":["Software & Cloud"],"capabilities":[],"topics":["Code"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22175","pdf":"https://arxiv.org/pdf/2605.22175","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.22175"},"evidence":{"snippet":"As a first step toward constructing high-quality test suites, we introduce SWE-Mutation, a benchmark for evaluating LLM-generated test suites.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.22175"},"ranking":{},"description":"SWE-Mutation evaluates LLM-generated test suites in software engineering by using 2,636 mutated variants derived from 800 original instances across nine programming languages, measuring verification and detection rates.","whyItMatters":"High-quality test suites are critical for program repair and reinforcement learning signals. SWE-Mutation reveals inadequacies in LLM-generated tests, guiding improvements in code generation and validation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-21T04:30:40.934319Z","inputHash":"e441cc8ae619179d87f2496a3f9d36c1468825e6fe540339831478355b235ff4"},"motivation":"Evaluating software engineering capabilities has become a core component of modern large language models (LLMs); however, the key bottleneck hindering further scaling lies not in the scarcity of high-quality solutions, but in the lack of high-quality test suites.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"publication_reported","venue":"ACL 2026 Findings","evidence":"ACL 2026 Findings","evidenceUrl":"https://arxiv.org/abs/2605.22175","source":"arxiv-journal-reference","evidenceLevel":"strong-author-metadata","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publications":[{"venueName":"ACL 2026 Findings","publicationStatus":"published","evidence":[{"sourceType":"arxiv-journal-reference","sourceUrl":"https://arxiv.org/abs/2605.22175","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"ACL 2026 Findings","level":"strong-author-metadata"}]}],"publishers":[{"name":"SWE-Mutation Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2605.22175","role":"benchmark-publisher"}],"capabilityGroups":["Coding & Software Engineering"],"domainScope":"general"},{"id":"bm_sgr-bench_d4ee6524","familyId":"bmf_3984c874ff5c","name":"SGR-Bench","oneLine":"SGR-Bench evaluates search agents on state-gated retrieval tasks. It includes 100 expert-curated tasks across 12 public data ecosystems, requiring agents to configure site-specific filters, views, hierarchies, or scopes to retrieve structured answers. Tasks come in goal-oriented and constraint-guided formulations.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.AI"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22219","pdf":"https://arxiv.org/pdf/2605.22219","project":null,"code":null,"data":"https://huggingface.co/datasets/PKUAIWeb/SGR-BENCH","hfPaper":"https://huggingface.co/papers/2605.22219"},"evidence":{"snippet":"We introduce SGR-Bench, a benchmark for this setting containing 100 expert-curated tasks spanning six source families and 12 public data ecosystems.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":190,"hfDatasetLikes":0},"source":{"type":"arxiv","id":"2605.22219"},"ranking":{},"description":"SGR-Bench evaluates search agents on state-gated retrieval tasks. It includes 100 expert-curated tasks across 12 public data ecosystems, requiring agents to configure site-specific filters, views, hierarchies, or scopes to retrieve structured answers. Tasks come in goal-oriented and constraint-guided formulations.","whyItMatters":"SGR-Bench addresses an undercharacterized class of retrieval tasks where evidence is hidden behind site-specific retrieval states. It provides a standardized evaluation to measure agents' ability to establish correct retrieval states, which is critical for real-world data retrieval from specialized websites.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"918576c8c087f4bc05fafe7baaf735ead5edcb0ab74b8f5ecfb8391d8c75d9a4"},"motivation":"Recent advances in large language models and tool-using agents have expanded the range of benchmarked web tasks.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.22219","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"PKUAIWeb","organizationType":"academic-lab","sourceUrl":"https://huggingface.co/datasets/PKUAIWeb/SGR-BENCH","role":"benchmark-publisher"}],"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_sdgbiasbench_aa7777ae","familyId":"bmf_7819f7afec07","name":"SDGBiasBench","oneLine":"SDGBiasBench is a benchmark suite for evaluating vision-language models on Sustainable Development Goals (SDG) reasoning, covering 500k multiple-choice questions and 50k regression tasks.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Multimodal","Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.21919","pdf":"https://arxiv.org/pdf/2605.21919","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.21919"},"evidence":{"snippet":"To address this gap, we propose SDGBiasBench, a large-scale benchmark suite for SDG-oriented vision-language reasoning.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.21919"},"ranking":{},"description":"SDGBiasBench is a benchmark suite for evaluating vision-language models on Sustainable Development Goals (SDG) reasoning, covering 500k multiple-choice questions and 50k regression tasks.","whyItMatters":"Existing SDG evaluation tools lack a combined assessment of qualitative and quantitative reasoning, and this benchmark aims to expose systematic biases in model predictions. The value lies in enabling more reliable AI for sustainable development monitoring.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"6dc257974a66feb822577a03a7b8377bdfba5ddf19c19f1218ec7b61d143cf99"},"motivation":"Assessing progress toward the Sustainable Development Goals (SDGs) requires multi-step reasoning over visual cues, contextual knowledge, and development indicators, where incomplete evidence use and imperfect evidence integration can introduce hidden prediction biases.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.21919","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_receiptbench_9f399a36","familyId":"bmf_268e28d589d5","name":"ReceiptBench","oneLine":"ReceiptBench evaluates multimodal large language models on visual information extraction from receipts. It includes 10k human-annotated receipts and four hierarchical subtasks: basic perception, format normalization, semantic reasoning, and structure parsing, with defined metrics and scoring.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22413","pdf":"https://arxiv.org/pdf/2605.22413","project":null,"code":"https://github.com/wwwT0ri/ReceiptBench","data":null,"hfPaper":"https://huggingface.co/papers/2605.22413"},"evidence":{"snippet":"To bridge this gap, we introduce ReceiptBench, a large-scale, human-annotated benchmark consisting of 10k diverse receipts, organizing information extraction into four hierarchical sub-tasks: (1) Basic Perception for raw text spotting, (2) Format Normalization for strictly following standardization instructions, (3) Semantic Reasoning for inferring implicit attributes from context, and (4) Structure Parsing for handling nested line items.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":3,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.22413"},"ranking":{},"description":"ReceiptBench evaluates multimodal large language models on visual information extraction from receipts. It includes 10k human-annotated receipts and four hierarchical subtasks: basic perception, format normalization, semantic reasoning, and structure parsing, with defined metrics and scoring.","whyItMatters":"Existing VIE benchmarks lack scale, realism, and semantic granularity. ReceiptBench provides a standardized, publicly available evaluation path for comparing models on diverse receipt understanding tasks, supporting practical deployment decisions in document automation.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"92f6009abe115058a572d105ce9d8bc1c0ec93cba81e4c0fff9da5794c395390"},"motivation":"Extracting structured information from visual documents (Visual Information Extraction, VIE) is a cornerstone of business automation.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.22413","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"T0rl (ReceiptBench team)","organizationType":"academic-lab","sourceUrl":"https://github.com/wwwT0ri/ReceiptBench","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_oncotriad-qa_ae032868","familyId":"bmf_f1efe287a4be","name":"OncoTriad-QA","oneLine":"OncoTriad-QA is a patient-level benchmark for pan-cancer question answering, integrating radiology, pathology, genomics, and clinical metadata from TCGA. It includes 86.1k questions across 9,281 cases and 32 cancer cohorts, with annotations derived from curated labels and diagnostic reports.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":["Reasoning"],"topics":["Reasoning"],"construction":"Transform Existing","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2608.02615","pdf":"https://arxiv.org/pdf/2608.02615","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2608.02615"},"evidence":{"snippet":"We introduce OncoTriad-QA, a patient-level radiology-pathology-genomics benchmark for pan-cancer question answering.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2608.02615"},"ranking":{},"description":"OncoTriad-QA is a patient-level benchmark for pan-cancer question answering, integrating radiology, pathology, genomics, and clinical metadata from TCGA. It includes 86.1k questions across 9,281 cases and 32 cancer cohorts, with annotations derived from curated labels and diagnostic reports.","whyItMatters":"Existing medical LLM benchmarks often focus on isolated modalities. OncoTriad-QA could address the gap in evaluating integrated, patient-level reasoning across multiple evidence types, which is crucial for real-world oncology decision support.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"9b349298381d03d3f706f8c6d7ed3c680ea5aa197cead9d8feec58605efcc2be"},"motivation":"Cancer diagnosis and characterization require integrating complementary evidence from radiology, pathology, genomics, and clinical metadata.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2608.02615","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"},{"id":"bm_mpdocbench-parse_f77fb798","familyId":"bmf_790789c36119","name":"MPDocBench-Parse","oneLine":"MPDocBench-Parse is a benchmark for multi-page document parsing, covering 433 manually annotated documents with 3,246 pages across 15 document types in English and Chinese. It evaluates content fidelity and logical structure, including text, table, formula recognition, reading order, and heading hierarchy recovery.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22100","pdf":"https://arxiv.org/pdf/2605.22100","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.22100"},"evidence":{"snippet":"To address these gaps, we propose MPDocBench-Parse, a benchmark for multi-page document parsing in real-world applications.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.22100"},"ranking":{},"description":"MPDocBench-Parse is a benchmark for multi-page document parsing, covering 433 manually annotated documents with 3,246 pages across 15 document types in English and Chinese. It evaluates content fidelity and logical structure, including text, table, formula recognition, reading order, and heading hierarchy recovery.","whyItMatters":"The benchmark addresses the lack of realistic multi-page document parsing evaluation, offering a more comprehensive protocol for assessing semantic continuity, hierarchical structure recovery, and visual content preservation, which is crucial for advancing document parsing systems in practical applications.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"71f565417af5043db56aedc917cbb88c3b877c75d07086ea7814f969a8b01ce2"},"motivation":"Document parsing converts visually rich documents into machine-readable structured representations, forming a crucial foundation for information systems.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.22100","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_jmed48k_34101c20","familyId":"bmf_cfe906a6cd3c","name":"JMed48k","oneLine":"JMed48k evaluates vision-language models on 48,862 Japanese medical licensing exam questions from 11 national examinations (2005-2025), with images annotated under an 8-type taxonomy. The JMed48k-Eval subset contains 12,484 scored questions, including text-only and with-image items, scored separately.","area":"Multimodal","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22080","pdf":"https://arxiv.org/pdf/2605.22080","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.22080"},"evidence":{"snippet":"We introduce JMed48k, a multi-profession Japanese healthcare licensing benchmark for evaluating vision-language models.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.22080"},"ranking":{},"description":"JMed48k evaluates vision-language models on 48,862 Japanese medical licensing exam questions from 11 national examinations (2005-2025), with images annotated under an 8-type taxonomy. The JMed48k-Eval subset contains 12,484 scored questions, including text-only and with-image items, scored separately.","whyItMatters":"Provides a profession-stratified evaluation for vision-language models in Japanese medical licensing, enabling comparison of image use across professions and model types, and addressing the lack of multilingual medical benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c18c24c9f88a6eb18bbd546b52bcd69f7581db89456374faed93534ee5cbb963"},"motivation":"We introduce JMed48k, a multi-profession Japanese healthcare licensing benchmark for evaluating vision-language models.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.22080","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"JMed48k Team","organizationType":"academic-lab","sourceUrl":"https://arxiv.org/abs/2605.22080","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"specific"},{"id":"bm_ishigaki-ids-bench_d58a0879","familyId":"bmf_4b152674589d","name":"Ishigaki-IDS-Bench","oneLine":"A benchmark for generating Information Delivery Specification (IDS) XML from BIM information requirements, with 166 examples in English and Japanese. It evaluates LLMs on formal validity via IDSAuditTool and content fidelity via facet-level macro-F1 against gold IDS files.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":0.85,"links":{"report":"https://arxiv.org/abs/2605.22079","pdf":"https://arxiv.org/pdf/2605.22079","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.22079"},"evidence":{"snippet":"We present Ishigaki-IDS-Bench, the first publicly released benchmark for IDS generation from BIM information requirements.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.22079"},"ranking":{},"description":"A benchmark for generating Information Delivery Specification (IDS) XML from BIM information requirements, with 166 examples in English and Japanese. It evaluates LLMs on formal validity via IDSAuditTool and content fidelity via facet-level macro-F1 against gold IDS files.","whyItMatters":"IDS authoring requires domain expertise and vocabulary conformance; this benchmark quantifies LLM performance on a structured generation task where output must satisfy external validation tools, filling a gap for capability assessment in BIM-specific language models.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"c27197e53cf8db2dad459e76339bd492abf8bcdc7e06d7a1555d7d6cf7f0931b"},"motivation":"Building Information Modeling (BIM) projects increasingly use Information Delivery Specification (IDS) to formalize information requirements in a machine-checkable XML format.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.22079","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_idiolink_97ac8121","familyId":"bmf_f52bc82b7c8c","name":"IdioLink","oneLine":"IdioLink is a retrieval benchmark with 10,700 documents and 2,140 queries across 107 idioms, testing the linking of idiomatic expressions to literal or paraphrased equivalents. Annotations mark core meaning spans, and scoring is based on retrieval quality against these spans.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Information retrieval"],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22247","pdf":"https://arxiv.org/pdf/2605.22247","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.22247"},"evidence":{"snippet":"We introduce IdioLink, a retrieval benchmark designed to test whether models can link idiomatic expressions to conceptually equivalent meanings expressed in literal or paraphrased forms.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.22247"},"ranking":{},"description":"IdioLink is a retrieval benchmark with 10,700 documents and 2,140 queries across 107 idioms, testing the linking of idiomatic expressions to literal or paraphrased equivalents. Annotations mark core meaning spans, and scoring is based on retrieval quality against these spans.","whyItMatters":"Existing retrieval models often rely on surface similarity and fail on idiomatic expressions. IdioLink provides a reusable testbed to measure semantic abstraction beyond lexical overlap, offering a practical way to evaluate idiom-aware retrieval systems.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"cd2be64596d629add0bf0d855d62f18c00dac323bf38b70e7ffb9d4351dfdbf0"},"motivation":"Idioms pose a fundamental challenge for language models, as their meaning cannot be inferred from surface form alone.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.22247","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning","Search & Retrieval"],"domainScope":"general"},{"id":"bm_hyperalign-bench_25471ecc","familyId":"bmf_6933ddf622b3","name":"HyperAlign-Bench","oneLine":"HyperAlign-Bench is a benchmark introduced in a paper for evaluating hypergraph structural modeling in large language models, covering vertex-level and hyperedge-level tasks.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CL"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.21858","pdf":"https://arxiv.org/pdf/2605.21858","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.21858"},"evidence":{"snippet":"To systematically evaluate different methods in hypergraph structural modeling, we introduce HyperAlign-Bench.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.21858"},"ranking":{},"description":"HyperAlign-Bench is a benchmark introduced in a paper for evaluating hypergraph structural modeling in large language models, covering vertex-level and hyperedge-level tasks.","whyItMatters":"It addresses the gap in evaluating LLMs on high-order relational structures, which are common in real-world data but underrepresented in existing benchmarks.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"bbee5089f77780139c899f01fbd5876b685c31f302c6389345630109906ca587"},"motivation":"Large language models (LLMs) have recently shown strong potential in modeling relational structures.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.21858","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_evalawarebench_6e7fbcdf","familyId":"bmf_3a8c0d6f9044","name":"EvalAwareBench","oneLine":"EvalAwareBench is a factor-controlled benchmark of 100 paired safety-capability tasks, each toggling one of eight evaluation-awareness triggers while holding the underlying request fixed. It measures recognition and behavioral change in language models under evaluation.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.23055","pdf":"https://arxiv.org/pdf/2605.23055","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.23055"},"evidence":{"snippet":"To study which factors each model is sensitive to and how they interact, we propose \\textbf{EvalAwareBench}, a factor-controlled benchmark of 100 paired safety-capability tasks where each of the eight factors can be independently toggled, varying evaluative signals while holding the underlying request fixed.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.23055"},"ranking":{},"description":"EvalAwareBench is a factor-controlled benchmark of 100 paired safety-capability tasks, each toggling one of eight evaluation-awareness triggers while holding the underlying request fixed. It measures recognition and behavioral change in language models under evaluation.","whyItMatters":"Evaluation awareness can distort benchmark results, especially for safety evaluations. EvalAwareBench enables controlled measurement of model sensitivity to evaluative signals, helping researchers identify and mitigate threats to benchmark validity.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"e8ec7897967b2d9aa9d0c4f85ae6c9690761e4b38b53f85bf00fb2685dc30bb3"},"motivation":"Frontier language models sometimes recognize that they are being evaluated and adjust their behavior, undermining validity of benchmark results.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.23055","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_csvfidelity-bench_89145848","familyId":"bmf_8cb890c5c077","name":"CSVFidelity-Bench","oneLine":"CSVFidelity-Bench evaluates knowledge graph construction from statistical tables, focusing on the effect of extraction schema and format coupling on fidelity. It includes 15 datasets, 11 Type-II and 4 Type-III tables, with 1,892 gold standard facts across 6 domains.","area":"Language & Knowledge","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":["Factuality"],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.21974","pdf":"https://arxiv.org/pdf/2605.21974","project":"https://anonymous.4open.science/r/sge_lightrag-BE19","code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.21974"},"evidence":{"snippet":"To support fidelity-aware evaluation, we release CSVFidelity-Bench.","reasonCodes":["exact named benchmark artifact released in abstract","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.21974"},"ranking":{},"description":"CSVFidelity-Bench evaluates knowledge graph construction from statistical tables, focusing on the effect of extraction schema and format coupling on fidelity. It includes 15 datasets, 11 Type-II and 4 Type-III tables, with 1,892 gold standard facts across 6 domains.","whyItMatters":"Addresses the gap in evaluating knowledge graph construction fidelity, particularly the overlooked interaction between serialization format and schema constraints, which can cause catastrophic coverage loss.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3dde2798e517b59f957714c3955f28c0ca2e59cf3ec6cd2539d2865c1f749af2"},"motivation":"An extraction schema should not reduce knowledge graph fidelity.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.21974","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"general"},{"id":"bm_cocotree_73046ddd","familyId":"bmf_81fdc618d821","name":"COCOTree","oneLine":"COCOTree evaluates open tree-structured visual decomposition, segmenting images into hierarchical trees of visual components with unconstrained granularity. It includes over 21K images and 1.8M structural nodes, with 3.5K unique labels, and uses the Open Tree Quality (OTQ) metric for scoring.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.CV"],"construction":"Unknown","annotation":"Unknown","readiness":"Runnable","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22068","pdf":"https://arxiv.org/pdf/2605.22068","project":null,"code":"https://github.com/melonkick3090/COCOTree","data":null,"hfPaper":"https://huggingface.co/papers/2605.22068"},"evidence":{"snippet":"We release our dataset and benchmark code at https://github.com/melonkick3090/COCOTree.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":0,"hfDailySubmittedAt":null,"githubStars":2,"githubScope":"benchmark_repo","hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.22068"},"ranking":{},"description":"COCOTree evaluates open tree-structured visual decomposition, segmenting images into hierarchical trees of visual components with unconstrained granularity. It includes over 21K images and 1.8M structural nodes, with 3.5K unique labels, and uses the Open Tree Quality (OTQ) metric for scoring.","whyItMatters":"Provides a standardized evaluation for a new task paradigm, enabling comparison across models on open vocabulary and long-tail visual structures, where existing benchmarks lack hierarchical decomposition metrics.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"033429e4f945fd68b342ba11b6424803e89a9109aec69387a01ee961b3d8faf0"},"motivation":"We formalize and enable the task of open tree decomposition, which segments an image into hierarchical trees of visual components with unconstrained granularity and flexibility.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.22068","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"publishers":[{"name":"Anonymous","organizationType":"academic-lab","sourceUrl":"https://github.com/melonkick3090/COCOTree","role":"benchmark-publisher"}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_boiling-the-frog_8414ba51","familyId":"bmf_f1d48d3c8c74","name":"Boiling the Frog","oneLine":"Boiling the Frog evaluates whether tool-using AI models in corporate settings are susceptible to incremental attacks through multi-turn scenarios with persistent workspaces. It includes a three-level operational risk taxonomy and scores attack success rate (ASR) on resulting artifact state.","area":"Safety & Trustworthiness","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Safety"],"construction":"Aggregate Existing","annotation":"Mixed","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22643","pdf":"https://arxiv.org/pdf/2605.22643","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.22643"},"evidence":{"snippet":"We introduce Boiling the Frog, a benchmark that evaluates whether tool-using AI models deployed in corporate and office settings are susceptible to incremental attacks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.22643"},"ranking":{},"description":"Boiling the Frog evaluates whether tool-using AI models in corporate settings are susceptible to incremental attacks through multi-turn scenarios with persistent workspaces. It includes a three-level operational risk taxonomy and scores attack success rate (ASR) on resulting artifact state.","whyItMatters":"Addresses the gap in safety evaluation for agents acting in environments, focusing on incremental manipulation rather than single-turn textual outputs.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"fcdac064900bc9e92b7202a5484c1389a775db9ac286461cf3194aed80de665c"},"motivation":"Background.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.22643","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Safety & Trustworthiness"],"domainScope":"general"},{"id":"bm_ateliereval_cc1f5b0c","familyId":"bmf_40ece635281f","name":"AtelierEval","oneLine":"AtelierEval evaluates prompting proficiency of humans and MLLMs for text-to-image systems via 360 tasks, using a skill-based agentic evaluator (AtelierJudge) that scores prompt-image pairs subjectively and objectively.","area":"Vision & 3D","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["cs.AI"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22645","pdf":"https://arxiv.org/pdf/2605.22645","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.22645"},"evidence":{"snippet":"We introduce AtelierEval, the first unified benchmark that quantifies prompting proficiency across 360 expert-crafted tasks.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.22645"},"ranking":{},"description":"AtelierEval evaluates prompting proficiency of humans and MLLMs for text-to-image systems via 360 tasks, using a skill-based agentic evaluator (AtelierJudge) that scores prompt-image pairs subjectively and objectively.","whyItMatters":"Introduces a new evaluation angle for T2I pipelines, measuring upstream prompting ability which is currently unassessed.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"849add50d73fcb34de0a5434ed011db8056f58f81af2ee4ea6b4c4d5062dea81"},"motivation":"Text-to-image (T2I) systems increasingly rely on upstream prompters, either humans or multimodal large language models (MLLMs), to translate user intent into detailed prompts.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"acceptance_claimed","venue":"ICML 2026","evidence":"Accepted by ICML 2026","evidenceUrl":"https://arxiv.org/abs/2605.22645","source":"arxiv-comments","evidenceLevel":"author-claim","verifiedAt":"2026-08-24T07:42:10.129552Z"},"venueAttempts":[{"venueName":"ICML 2026","reviewStatus":"accepted","decisionRaw":"Accepted by ICML 2026","evidence":[{"sourceType":"arxiv-comments","sourceUrl":"https://arxiv.org/abs/2605.22645","observedAt":"2026-08-24T07:42:10.129552Z","rawValue":"Accepted by ICML 2026","level":"author-claim"}]}],"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_agrotools_ca69be78","familyId":"bmf_c721caf5d4ed","name":"AgroTools","oneLine":"AgroTools evaluates tool-augmented multimodal agents in agriculture, with 539 QA instances, 1,097 images, 14 executable tools, and structured tool-use traces for process and outcome evaluation.","area":"Multimodal","applicationDomains":[],"primaryDomain":"General AI","industrySectors":[],"capabilities":[],"topics":["Multimodal"],"construction":"Unknown","annotation":"Unknown","readiness":"Inspectable","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.22366","pdf":"https://arxiv.org/pdf/2605.22366","project":null,"code":null,"data":"https://huggingface.co/datasets/AgroTools/AgroTools","hfPaper":"https://huggingface.co/papers/2605.22366"},"evidence":{"snippet":"In this paper, we introduce AgroTools, a benchmark for evaluating tool-augmented multimodal agents in agriculture.","reasonCodes":["exact coined title identity tied to benchmark evidence","benchmark release stated in one sentence","evaluation protocol evidence","public artifact URL"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":null,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":57,"hfDatasetLikes":2},"source":{"type":"arxiv","id":"2605.22366"},"ranking":{},"description":"AgroTools evaluates tool-augmented multimodal agents in agriculture, with 539 QA instances, 1,097 images, 14 executable tools, and structured tool-use traces for process and outcome evaluation.","whyItMatters":"Fills a gap in agricultural benchmarks by assessing tool use and workflow execution, not just final answers, enabling deeper evaluation of agent capabilities.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"3a593c4aa01e277c30178213cb4c9e9f303445f60b0cf9175cd4500fd396d304"},"motivation":"Agricultural decision-making increasingly requires multimodal systems that can transform visual observations into reliable, executable actions.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.22366","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Multimodal Perception"],"domainScope":"general"},{"id":"bm_a-large-scale-dataset-and-benchmark_80e45544","familyId":"bmf_06e6d8e2fe19","name":"A Large-Scale Dataset and Benchmark","oneLine":"InteractBind provides a large-scale dataset of ~100k protein-ligand pairs with fine-grained binding-site localization tasks and interaction maps for six non-covalent interaction types, plus affinity and similarity-controlled splits.","area":"Language & Knowledge","applicationDomains":["Health & Life Sciences"],"primaryDomain":"Health & Life Sciences","industrySectors":["Pharma & Biotech"],"capabilities":[],"topics":["cs.LG"],"construction":"Unknown","annotation":"Unknown","readiness":"Paper only","releasedAt":"2026-05-21","firstSeenAt":"2026-08-19","recognitionConfidence":1.0,"links":{"report":"https://arxiv.org/abs/2605.24045","pdf":"https://arxiv.org/pdf/2605.24045","project":null,"code":null,"data":null,"hfPaper":"https://huggingface.co/papers/2605.24045"},"evidence":{"snippet":"To address this gap, we introduce InteractBind, a large-scale protein-ligand dataset comprising approximately 100k protein-ligand pairs, together with a benchmark for fine-grained evaluation.","reasonCodes":["coined title prefix ending in Bench or Benchmark","benchmark release stated in one sentence","evaluation protocol evidence"]},"dataStatus":"primary-source-indexed","demo":false,"attention":{"asOf":"2026-08-24","hfPaperUpvotes":1,"hfDailySubmittedAt":null,"githubStars":null,"githubScope":null,"hfDatasetDownloads":null,"hfDatasetLikes":null},"source":{"type":"arxiv","id":"2605.24045"},"ranking":{},"description":"InteractBind provides a large-scale dataset of ~100k protein-ligand pairs with fine-grained binding-site localization tasks and interaction maps for six non-covalent interaction types, plus affinity and similarity-controlled splits.","whyItMatters":"Challenges existing protein-ligand benchmarks by focusing on localization and interaction interpretability, which is critical for drug discovery.","copyGeneration":{"model":"deepseek-v4-flash","generatedAt":"2026-08-20T15:22:10.044731Z","inputHash":"d9fc988423d4f52a06e3da234630e597ddf74eca1f6a54f647a6ba78aa296eb2"},"motivation":"Protein-ligand modeling underpins computational drug discovery and molecular design.","constructionDetail":"Unknown — the indexer does not infer construction details without explicit source evidence.","publication":{"status":"unverified","venue":null,"evidence":null,"evidenceUrl":"https://arxiv.org/abs/2605.24045","source":"arxiv-metadata","evidenceLevel":"unverified","verifiedAt":"2026-08-24T07:42:10.129552Z"},"capabilityGroups":["Knowledge & Reasoning"],"domainScope":"specific"}]}