{"schemaVersion":"1.0","version":"2026-09-27.1","checkedAt":"2026-09-27","roles":[{"id":"DEV","name":"AI开发","summary":"从明确需求到可验收的软件交付","workflow":"定位问题 → 修改实现 → 回归 → 发布判断"},{"id":"PM","name":"AI产品","summary":"把用户问题转成有依据的产品决策","workflow":"需求与机会 → 交互规格 → 发布判断 → 商业实验"},{"id":"FDE","name":"企业交付 / FDE","summary":"让Agent进入真实业务流程","workflow":"流程发现 → 系统集成 → 知识辅助 → 采用与交接"},{"id":"DATA","name":"数据分析","summary":"从业务口径到可复核的分析结论","workflow":"口径确认 → 数据对账 → 变化分析 → 实验审核"},{"id":"GROW","name":"内容与增长","summary":"从可信内容到可解释的增长判断","workflow":"分群内容 → 渠道适配 → 激活实验 → 增量与收益"}],"tasks":[{"id":"DEV-01","role":"DEV","title":"受限接口幂等修复","version":"0.1.0","trigger":"已复现重复操作缺陷","inputs":["代码快照","业务规格","失败日志"],"outputs":["补丁","反例与回归","影响和恢复说明"],"acceptance":["同key同payload不重复","不同payload冲突","失败可恢复"],"exception":"响应丢失、并发、key生命周期不明时澄清实现范围","human":"确定业务操作身份与合并决定","evidence":["E01","E02","E15"],"coverage":"仓库修复、回归、缺信息提问","gap":"未找到同一幂等规格逐题对应；业务操作身份由现场规则决定","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"DEV-02","role":"DEV","title":"业务写入与事件交付一致性","version":"0.1.0","trigger":"业务已写但下游未收到事件","inputs":["数据库/消息契约","故障注入环境"],"outputs":["原子outbox","重放与去重","维护说明"],"acceptance":["提交与待发事件一致","消费者幂等","失败不静默丢失"],"exception":"跨服务副作用需补偿；长时间失败进入异常队列","human":"决定一致性、补偿和上线责任","evidence":["E01","E03"],"coverage":"代码修改及运行环境任务族","gap":"分布式一致性、恢复语义的专门覆盖未审定","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"DEV-03","role":"DEV","title":"版本与权限约束的检索回答","version":"0.1.0","trigger":"用户提出业务知识问题","inputs":["可访问文档","版本","权限","题集"],"outputs":["回答服务","引用","拒绝/升级记录"],"acceptance":["引用支持结论","无权内容不可检出","过期冲突可见"],"exception":"缺依据时不补造答案，转澄清/人工","human":"定义授权、知识有效性及可接受错误","evidence":["E08","E09"],"coverage":"知识查询、规则遵循","gap":"跨版本检索与细粒度访问控制尚无直接匹配","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"DEV-04","role":"DEV","title":"工具Agent评测与回归诊断","version":"0.1.0","trigger":"模型/工具版本变化或线上失败","inputs":["轨迹","最终状态","工具合同","固定题集"],"outputs":["分层评测","故障归因","修复与回归"],"acceptance":["最终业务状态核对","预算与异常行为达标","保留失败分母"],"exception":"工具报成功但状态未达成须标失败","human":"审验收定义、上线范围、归因不确定","evidence":["E03","E09","E15","E19","E21"],"coverage":"工具执行、交互可靠性、求助判分协议","gap":"这些benchmark评价Agent，未证明Agent能正确设计评测","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"PM-01","role":"PM","title":"有证据的AI机会判断","version":"0.1.0","trigger":"业务提出AI需求","inputs":["访谈","工单","现状流程","成本数据"],"outputs":["证据矩阵","备选方案","可证伪试点假设"],"acceptance":["事实与推断分离","非AI方案比较","基线明确"],"exception":"相互矛盾或缺现场材料时留unknown","human":"选择目标和取舍，不编造用户需求","evidence":["E11","E04","E18"],"coverage":"需求抽取、专业分析、咨询任务","gap":"抽取是子任务；机会价值和资源取舍无直接验证","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"PM-02","role":"PM","title":"带失败恢复的人机交互规格","version":"0.1.0","trigger":"准备验证AI工作流原型","inputs":["任务","用户","权限","失败案例"],"outputs":["原型","验收","纠错/撤销/升级说明"],"acceptance":["能力边界可理解","失败可发现和接管","责任接收明确"],"exception":"不可逆动作无授权不纳入默认自动化","human":"设定用户控制和组织责任","evidence":["E15","E09"],"coverage":"澄清和失败场景可作为规格依据","gap":"交互benchmark不是规格编写能力测验","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"PM-03","role":"PM","title":"模型版本发布决策","version":"0.1.0","trigger":"新版本总评测分数上升","inputs":["切片成绩","错误严重度","延迟成本","线上依据"],"outputs":["限域发布/影子/拒绝建议与回滚条件"],"acceptance":["关键错误单列","可靠性成本合格","离线不冒充线上"],"exception":"样本无效则不判效果；无现场证据仅影子","human":"确定阈值及上线范围","evidence":["E01","E03","E15"],"coverage":"比较评测版本与配置的方法证据","gap":"发布决策和承担风险未被测试","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"PM-04","role":"PM","title":"AI功能商业实验判读","version":"0.1.0","trigger":"试点结束需要继续或停止","inputs":["分流曝光","主指标护栏","成本","观察窗"],"outputs":["有效性判定","效果区间","单位经济与后续"],"acceptance":["SRM/污染审查","分母正确","不挑指标报喜"],"exception":"非显著或样本不足保留不确定","human":"业务收益与错误代价取舍","evidence":["E13","E18"],"coverage":"历史实验素材与辅助判断研究","gap":"未确认AI判读该本地商业实验的直接结果","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"FDE-01","role":"FDE","title":"客户流程发现与试点契约","version":"0.1.0","trigger":"客户希望AI改造流程","inputs":["现场观察/授权记录","SOP","系统清单"],"outputs":["现状与例外图","试点范围","验收/停止和责任表"],"acceptance":["客户确认问题与基线","系统权限/接收人明确"],"exception":"SOP与实际不同保留差异，不默认任一正确","human":"与客户确认问题、预算和责任","evidence":["E11","E15"],"coverage":"需求抽取、关键缺口识别","gap":"现场访谈、谈判和试点承诺未覆盖","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"FDE-02","role":"FDE","title":"受限业务系统集成与交接","version":"0.1.0","trigger":"客户批准在副本验证流程","inputs":["接口","对象字典","权限","样例记录"],"outputs":["集成","对账","故障/撤销","运维交接"],"acceptance":["对象映射正确","写入可追溯","重放不重复","负责人可接管"],"exception":"权限撤回、部分成功、后台绕行写入","human":"客户系统所有者批准上线；FDE负责交付事实","evidence":["E08","E09","E10","E03","E21","E22","E23"],"coverage":"系统操作、业务工具与交互","gap":"UI操作不等于系统集成；交接与客户验收未覆盖","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"FDE-03","role":"FDE","title":"设备知识维护辅助","version":"0.1.0","trigger":"一线查询历史故障和适用资料","inputs":["设备型号","文档版本","单位","历史工单"],"outputs":["有出处且限定适用性的建议或澄清"],"acceptance":["型号/版本/单位匹配","未经授权不写设备状态"],"exception":"相似文档不适配；实时状态未知；需资格人员确认","human":"负责适用性审核和现场操作决定","evidence":["E08","E09"],"coverage":"知识查询、规则与工具子能力","gap":"设备专业正确性和危险处置未验证","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"FDE-04","role":"FDE","title":"现场采用与交接判定","version":"0.1.0","trigger":"影子阶段到期需要决定扩展","inputs":["使用资格","采纳/拒绝","返工","维护和培训记录"],"outputs":["继续/修改/停止建议，维护及培训包"],"acceptance":["阶段按事实","采用有分母","对业务后果与维护明确"],"exception":"只有宣传或部署计划不能认定规模化","human":"客户业务接收者与维护人共同验收","evidence":["E16"],"coverage":"观察实际授权与介入可作背景","gap":"无本任务直接benchmark，不能借用自主性指标代替采用","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"DATA-01","role":"DATA","title":"订单净额查询与对账","version":"0.1.0","trigger":"业务需报告交易净额","inputs":["订单","付款","退款","商品","口径及截止日"],"outputs":["SQL","逐单对账与数据质量记录"],"acceptance":["粒度和截止明确","不重复计数","取消/退款正确"],"exception":"多商品、分次付款、迟到退款","human":"确认业务口径与来源完整性","evidence":["E06","E07","E14","E20"],"coverage":"业务规则、多源计算；营销归因/账目类任务","gap":"具体退款口径与端到端对账未逐题对齐","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"DATA-02","role":"DATA","title":"跨工具指标契约","version":"0.1.0","trigger":"多个报表数字冲突","inputs":["现有SQL","业务定义","实体和时间字段"],"outputs":["指标字典","数据模型","质量检查与血缘"],"acceptance":["分子分母/单位/时区/过滤明确","变更可追溯"],"exception":"业务口径实质不同不强行同名统一","human":"与指标所有者确定含义","evidence":["E06","E07","E15"],"coverage":"业务文档、schema与澄清","gap":"组织指标语义共识不由SQL成绩证明","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"DATA-03","role":"DATA","title":"转化变化与组成分解","version":"0.1.0","trigger":"总体转化率出现变化","inputs":["分渠道或人群计数","前后时间窗"],"outputs":["总体/组内/组成分析，因果限制与建议"],"acceptance":["计数分母准确","未知不填0","描述不冒充因果"],"exception":"空分母、小样本、构成迁移","human":"解释统计结果并决定需要什么新数据","evidence":["E06","E12","E14","E20"],"coverage":"指标比较、分组和广告分析","gap":"因果识别与组成分解覆盖需逐题核对","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"DATA-04","role":"DATA","title":"实验有效性与区间审核","version":"0.1.0","trigger":"需要评估产品/运营干预","inputs":["分配","曝光","结果","指标预登记","数据时间"],"outputs":["有效性","效应区间和有限结论"],"acceptance":["SRM/污染先查","随机单位正确","区间与主指标对应"],"exception":"部分曝光不当作实际处理效果；显著性跨指标搬用","human":"判断因果前提与业务后果","evidence":["E13","E18"],"coverage":"实验可靠性方法与真实行为档案","gap":"尚无直接验证AI审核统计区间的匹配结果","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"GROW-01","role":"GROW","title":"证据驱动的分群内容","version":"0.1.0","trigger":"计划向某受众说明产品价值","inputs":["产品事实","研究/案例","目标受众"],"outputs":["内容草稿","主张台账","来源与限制"],"acceptance":["事实可追溯","实绩/预测分离","无虚构经历"],"exception":"来源失效或口径冲突需删减主张","human":"保证产品事实和发布责任","evidence":["E14","E04","E05"],"coverage":"有材料约束的文案和专业产物","gap":"分群增量与事实全部正确不能由总分推断","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"GROW-02","role":"GROW","title":"渠道与商业属性适配","version":"0.1.0","trigger":"内容需进入搜索或具体渠道","inputs":["规范","关键产品属性","用户意图","素材"],"outputs":["不同渠道版本及保留属性核对"],"acceptance":["关键属性不丢、不新增承诺","CTA适合受众"],"exception":"流畅改写丢型号/材料/限制","human":"判断语义、品牌与渠道适用性","evidence":["E14","E23"],"coverage":"platform-limits-gauntlet等官网任务名","gap":"官方任务级结果可引用，资产本轮未取到；真实渠道效果未测","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"GROW-03","role":"GROW","title":"分群激活实验设计","version":"0.1.0","trigger":"需要提高首次价值体验或留存","inputs":["行为事件","人群资格","预算","触达规则"],"outputs":["实验","触发/抑制","观察窗","停止规则"],"acceptance":["随机单位稳定","抑制符合明确约定","目标/护栏预设"],"exception":"已转化/无资格用户不计入目标触达","human":"选择受众、渠道与预算","evidence":["E14","E13"],"coverage":"factorial-ad-test等任务、历史实验方法","gap":"实际实验执行、分群代表性及激活效果未覆盖","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"},{"id":"GROW-04","role":"GROW","title":"增量与成本后收益判断","version":"0.1.0","trigger":"活动结束或预算调整","inputs":["随机分组","曝光","收入/贡献","成本","留存退款"],"outputs":["增量解释","净贡献","继续/停止与限制"],"acceptance":["因果对照有效","归因不等于增量","成本与业务护栏完整"],"exception":"点击上涨但净贡献下降；销售不显著","human":"对预算、用户体验和品牌取舍","evidence":["E12","E14","E13"],"coverage":"广告计算、归因敏感性、实验素材","gap":"归因不等于因果增量；长期利润无直接证明","localScore":null,"fieldValidated":false,"allocationStatus":"本站建议 · 待场景验证","checkedAt":"2026-09-27"}],"sources":[{"id":"E01","name":"SWE-Bench Pro V2","roles":["DEV"],"url":"https://labs.scale.com/leaderboard/swe_bench_pro_public_v2?tab=full","version":"V2; 2026-09-22","family":"Scale / Reflection","kind":"benchmark","supports":"修复与回归","limitation":"不能把聚合分数下放到幂等任务","access":"public_tasks_and_results","readDepth":"official_method_and_result_entry","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E02","name":"SWE-Lancer","roles":["DEV","PM"],"url":"https://openai.com/index/swe-lancer/","version":"2025-07 update","family":"OpenAI","kind":"benchmark","supports":"开发交付与技术方案选择","limitation":"历史结果，非完整产品管理","access":"public_split_and_results","readDepth":"official_research_summary","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E03","name":"Terminal-Bench","roles":["DEV","FDE"],"url":"https://www.tbench.ai/news/terminal-bench-4-0","version":"4.0.0","family":"Terminal-Bench / Harbor","kind":"benchmark","supports":"终端执行和环境验证","limitation":"跨大版本不可直接比较","access":"public_tasks_and_results","readDepth":"official_release","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E04","name":"GDPval","roles":["PM","DATA","GROW","DEV"],"url":"https://openai.com/index/gdpval/","version":"2025-09-25 introduction","family":"OpenAI","kind":"benchmark","supports":"专业产物验收","limitation":"一次性交付，不能全岗外推","access":"partial_public_tasks_and_results","readDepth":"official_method_and_results","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E05","name":"Remote Labor Index","roles":["PM","GROW","DEV","DATA"],"url":"https://labs.scale.com/leaderboard/rli","version":"live_page_checked_2026-09-27","family":"Scale / CAIS","kind":"benchmark","supports":"完整项目产物","limitation":"排除直接客户互动和长期效果","access":"private_scored_set_public_examples","readDepth":"official_method_and_result_entry","checkedAt":"2026-09-27","nextReview":"2026-10-27","localReproduced":false},{"id":"E06","name":"DABstep","roles":["DATA"],"url":"https://adyen-dabstep.hf.space/","version":"v1; validated submissions closed; v2 announced","family":"Adyen / Hugging Face","kind":"benchmark","supports":"多源业务数据分析","limitation":"Validated和Unvalidated分开，未抄取当前完整数值表","access":"public_tasks_external_scoring","readDepth":"official_method_and_status","checkedAt":"2026-09-27","nextReview":"2026-10-27","localReproduced":false},{"id":"E07","name":"Spider 2.0","roles":["DATA"],"url":"https://spider2-sql.github.io/","version":"2.0; track_required","family":"Spider研究团队","kind":"benchmark","supports":"企业SQL工作流","limitation":"输出正确不等于业务口径合理","access":"public_tasks_and_results","readDepth":"official_method_and_result_entry","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E08","name":"WorkArena / WorkArena++","roles":["FDE"],"url":"https://github.com/ServiceNow/WorkArena","version":"L1 / ++; commit_not_pinned","family":"ServiceNow研究团队","kind":"benchmark","supports":"企业UI和组合流程","limitation":"不证明客户发现或现场采用","access":"public_code_environment_access_required","readDepth":"official_repository","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E09","name":"tau2-bench","roles":["FDE","DEV"],"url":"https://github.com/sierra-research/tau2-bench","version":"domain_and_commit_required","family":"Sierra研究团队","kind":"benchmark","supports":"政策工具用户交互","limitation":"模拟用户不是实际客户；版本不同不合并","access":"public_simulated_benchmark","readDepth":"official_repository","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E10","name":"CRMArena-Pro","roles":["FDE","GROW"],"url":"https://arxiv.org/abs/2505.18878v1","version":"v1; 2025-05-24","family":"Salesforce研究团队","kind":"benchmark","supports":"CRM业务与多轮交互","limitation":"仓库限定研究用途，GUI需申请","access":"research_restricted_assets_and_results","readDepth":"paper_abstract_and_repository","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E11","name":"Voice of Customer","roles":["PM","FDE"],"url":"https://arxiv.org/abs/2503.01870v2","version":"v2; 2026-04-08","family":"Voice of Customer研究团队","kind":"study","supports":"客户需求抽取","limitation":"抽取不等于优先级；未确认完整数据开放","access":"published_study_dataset_access_unconfirmed","readDepth":"paper_selected_sections","checkedAt":"2026-09-27","nextReview":"2026-12-26","localReproduced":false},{"id":"E12","name":"AD-Bench","roles":["GROW","DATA"],"url":"https://arxiv.org/html/2602.14257v2","version":"v2; 2026-06-22","family":"AD-Bench研究团队","kind":"benchmark","supports":"广告分析正确性与轨迹","limitation":"非实际广告增量；旧资产核验显示数据未开放","access":"published_results_private_task_data","readDepth":"paper_methods_and_table2","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E13","name":"Upworthy Research Archive","roles":["GROW","DATA","PM"],"url":"https://upworthy.natematias.com/2024-06-upworthy-archive-update.html","version":"2024 reliability correction","family":"Upworthy档案研究团队","kind":"observation","supports":"真实实验结果与可靠性","limitation":"非当前AI评测；随机化异常需筛选","access":"historical_experiment_archive","readDepth":"maintainer_correction","checkedAt":"2026-09-27","nextReview":"2026-12-26","localReproduced":false},{"id":"E14","name":"Marketing Benchmark","roles":["GROW"],"url":"https://marketingbenchmark.ai/","version":"site run dated 2026-09-04","family":"Magister供应商","kind":"vendor","supports":"营销产物与分析任务","limitation":"单次尝试，自测，judge与holdout限制","access":"vendor_results_repository_fetch_404","readDepth":"vendor_method_and_visible_results","checkedAt":"2026-09-27","nextReview":"2026-10-27","localReproduced":false},{"id":"E15","name":"HiL-Bench","roles":["DEV","DATA","FDE","PM"],"url":"https://labs.scale.com/leaderboard/hil","version":"paper v4; 2026-05-04","family":"Scale研究团队","kind":"benchmark","supports":"选择性求助","limitation":"受控任务不证明现场最优分工","access":"public_tasks_and_results","readDepth":"official_method_abstract_data_card","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E16","name":"Anthropic Agent autonomy","roles":["DEV","FDE"],"url":"https://www.anthropic.com/research/measuring-agent-autonomy","version":"2026-02-18","family":"Anthropic","kind":"observation","supports":"授权打断澄清模式","limitation":"观察相关性及厂商用户样本","access":"observational_report","readDepth":"research_full_web_text","checkedAt":"2026-09-27","nextReview":"2026-12-26","localReproduced":false},{"id":"E17","name":"METR developer productivity update","roles":["DEV"],"url":"https://metr.org/blog/2026-02-24-uplift-update/","version":"2026-02-24","family":"METR","kind":"study","supports":"协作总时间与实验选择偏差","limitation":"后续结果不是可靠当前收益估计","access":"experiment_report_with_design_limitations","readDepth":"research_full_web_text","checkedAt":"2026-09-27","nextReview":"2026-12-26","localReproduced":false},{"id":"E18","name":"HBS BCG jagged frontier","roles":["PM","GROW"],"url":"https://aiinstitute.hbs.edu/navigating-the-jagged-technological-frontier/","version":"2023-09-21","family":"HBS / BCG","kind":"study","supports":"不同任务的辅助效果","limitation":"历史模型和咨询任务","access":"experiment_summary","readDepth":"research_institution_summary","checkedAt":"2026-09-27","nextReview":"2026-12-26","localReproduced":false},{"id":"E19","name":"AgentRx","roles":["DEV","FDE"],"url":"https://www.microsoft.com/en-us/research/blog/systematic-debugging-for-ai-agents-introducing-the-agentrx-framework/","version":"2026-03-12","family":"Microsoft Research","kind":"benchmark","supports":"Agent失败定位和根因诊断","limitation":"115条失败轨迹；诊断准确性不等于修复成功","access":"public_failure_trajectories","readDepth":"official_report_and_repository","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E20","name":"SpreadsheetBench 2","roles":["DATA","PM","GROW"],"url":"https://spreadsheetbench.github.io/","version":"2; 2026-06","family":"SpreadsheetBench研究团队","kind":"benchmark","supports":"工作簿生成、修复与可视化交付","limitation":"V1和V2分开；复杂环境及模型配置影响结果","access":"public_tasks_and_results","readDepth":"official_method_and_repository","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E21","name":"MCP Atlas","roles":["DEV","FDE","DATA"],"url":"https://labs.scale.com/leaderboard/mcp_atlas","version":"2026-04 revised protocol","family":"Scale研究团队","kind":"benchmark","supports":"工具发现、跨服务组合与错误恢复","limitation":"页首最高分与结果表不一致，不采用页首排名；不是客户系统集成","access":"partial_public_tasks_and_results","readDepth":"official_method_and_result_table","checkedAt":"2026-09-27","nextReview":"2026-10-27","localReproduced":false},{"id":"E22","name":"TheAgentCompany","roles":["FDE","PM","DEV"],"url":"https://github.com/TheAgentCompany/TheAgentCompany","version":"paper 2024 / NeurIPS 2025","family":"TheAgentCompany研究团队","kind":"benchmark","supports":"企业多应用和同事交互任务","limitation":"模拟公司及同事，非真人组织采用；未取当前榜单数值","access":"public_simulated_company","readDepth":"official_repository_and_paper_abstract","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false},{"id":"E23","name":"OSWorld 2.0","roles":["FDE","DATA","GROW"],"url":"https://osworld-v2.xlang.ai/","version":"2.0; released 2026-06-26","family":"OSWorld研究团队","kind":"benchmark","supports":"长流程、跨应用、状态与核对","limitation":"108任务及500步协议，不能与V1直接比较","access":"public_tasks_and_results","readDepth":"official_method_and_result_summary","checkedAt":"2026-09-27","nextReview":"2026-10-27","localReproduced":false},{"id":"E24","name":"SentinelBench","roles":["DEV","FDE"],"url":"https://www.microsoft.com/en-us/research/articles/sentinelbench-a-benchmark-for-long-running-monitoring-agents/","version":"2026-06-08","family":"Microsoft Research","kind":"benchmark","supports":"长期监测、等待、条件触发与不行动","limitation":"100个合成网页任务，事件时间人工设置；非生产稳定性证明","access":"public_synthetic_web_tasks","readDepth":"official_technical_summary","checkedAt":"2026-09-27","nextReview":"2026-11-26","localReproduced":false}],"results":[{"id":"R-AD-01","source":"E12","subject":"Claude-Opus-4.7","metric":"Pass@1","value":"76.9%","scope":"AD-Bench v2 · 225个广告分析任务","config":"论文所报Agent配置；整体任务集","evaluationDate":null,"reportedAt":"2026-06-22","locator":"论文 Table 2 / Overall","note":"单次回答正确率；L3为61.4%。不能代表真实广告增量。","status":"研究作者报告"},{"id":"R-CRM-01","source":"E10","subject":"论文中的领先Agent（摘要汇总）","metric":"单轮 / 多轮成功率","value":"约58% / 约35%","scope":"CRMArena-Pro 2025论文摘要","config":"多个Agent汇总；非单一配置比较","evaluationDate":null,"reportedAt":"2025-05-24","locator":"论文 Abstract","note":"多轮交互带来额外难度；历史结果，不作为当前排名。","status":"研究摘要"},{"id":"R-MKT-01","source":"E14","subject":"Magister + Gemini 3.8 Flash high effort","metric":"任务通过","value":"14 / 20","scope":"Marketing Benchmark · 页面2026-09-04运行","config":"完整Agent＋skills＋工具；每题一次","evaluationDate":"2026-09-04","reportedAt":"2026-09-04","locator":"官网 + Magister 表格","note":"官网自报；质量分94.5与整题通过不同。仓库访问404，尚未独立核实运行资产。","status":"供应商自测"},{"id":"R-OS-01","source":"E23","subject":"Claude Opus 4.8 · max thinking","metric":"整题完成 / 部分得分","value":"20.6% / 54.8%","scope":"OSWorld 2.0 · 108个长流程任务","config":"500步；批量工具调用","evaluationDate":null,"reportedAt":"2026-06-26","locator":"官方项目页 Introduction","note":"两种指标不同；部分分不能当工作完成率，不能与V1直接比较。","status":"研究作者报告"},{"id":"R-RX-01","source":"E19","subject":"AgentRx，相对提示基线","metric":"失败定位准确性提升","value":"+23.6个百分点","scope":"115条人工标注失败轨迹","config":"研究所报诊断流程与提示基线","evaluationDate":null,"reportedAt":"2026-03-12","locator":"Microsoft Research / Key Results","note":"是诊断准确性差值，不是任务成功率或修复成功率。","status":"研究作者报告"}],"changes":[{"id":"C02","date":"2026-09-27","kind":"来源核查","title":"DABstep区分已验证与未验证结果","source":"E06","before":"介绍页的早期结果容易被误作当前能力","after":"已验证榜单暂停新增验证；v2待发布","impact":"只引用具体结果状态和日期；没有取到可靠最新数值时保留原始入口。日期为本站核查日。","tasks":["DATA-01","DATA-02","DATA-03"]},{"id":"C05","date":"2026-09-27","kind":"来源核查","title":"营销自测保留可复核性缺口","source":"E14","before":"官网宣称任务、评分与原始运行已公开","after":"本轮访问所链仓库返回404","impact":"自测结果附明显限制，不升级为独立验证；后续重查公开资产。日期为本站核查日。","tasks":["GROW-01","GROW-02","GROW-03","GROW-04"]},{"id":"C01","date":"2026-09-22","kind":"评测修订","title":"SWE-Bench Pro修订任务与运行协议","source":"E01","before":"旧公开集731题","after":"V2为642题，修正任务并锁定协议","impact":"开发任务保留基准版本。旧版与新版成绩分开解释，不自动认定能力回退或进步。","tasks":["DEV-01","DEV-02"]},{"id":"C03","date":"2026-06-26","kind":"新增工作流","title":"OSWorld 2.0补充长流程与跨应用证据","source":"E23","before":"主要参考单应用或较短操作任务","after":"新增108个长流程任务与整题/部分评分","impact":"企业交付页加入状态恢复、跨来源核对和最终验收要求，仍不宣称现场采用已验证。","tasks":["FDE-02","GROW-02"]},{"id":"C04","date":"2026-03-12","kind":"新增方法","title":"失败诊断也有可引用的评测","source":"E19","before":"仅从是否完成观察Agent表现","after":"引入失败定位与根因分类证据","impact":"DEV-04增加失败轨迹审查；诊断增益与修复完成度分开展示。","tasks":["DEV-04"]}]}