[{"data":1,"prerenderedAt":2121},["ShallowReactive",2],{"blog-posts":3,"content-\u002Fposts\u002Fspinning-up-rl-intro-2":202},[4,16,21,26,37,43,47,59,66,72,76,81,89,96,104,109,115,121,126,131,138,145,152,158,165,173,179,185,191,197],{"path":5,"title":6,"description":7,"date":8,"categories":9,"tags":10,"draft":15},"\u002Fposts\u002Fspinning-up-policy-optimization","译","OpenAI Spinning Up 教程第三部分的全文翻译：推导最简单的策略梯度、log-derivative trick、EGLP 引理、reward-to-go 与基线，配 PyTorch 示例代码。","2026-09-28 17:49:25","技术",[11,12,13,14],"强化学习","RL","Spinning Up","翻译",false,{"path":17,"title":6,"description":18,"date":19,"categories":9,"tags":20,"draft":15},"\u002Fposts\u002Fspinning-up-rl-intro-2","OpenAI Spinning Up 第二部分的全文翻译：无模型与基于模型的 RL、策略优化与 Q-learning 的取舍，以及规划、专家迭代和经验增强等方法。","2026-09-28 10:20:21",[11,12,13,14],{"path":22,"title":6,"description":23,"date":24,"categories":9,"tags":25,"draft":15},"\u002Fposts\u002Fspinning-up-rl-intro","OpenAI Spinning Up 教程第一部分的全文翻译：智能体与环境、状态与观测、动作空间、策略、轨迹、奖励与回报、价值函数、贝尔曼方程、优势函数，附 MDP 形式化。","2026-09-28 09:29:40",[11,12,13,14],{"path":27,"title":28,"description":29,"date":30,"categories":31,"tags":32,"draft":36},"\u002Fposts\u002Fwhen-machines-come-alive","冰箱之后，生活还会怎样改变？","从厨房里的果蝇想到冰箱、移动互联网与 AI 的多轮调用：AGI 已经到来，只是昂贵的智能会怎样改变生活？","2026-09-25 06:36:22","生活杂谈",[33,34,35],"AI","随笔","科技",true,{"path":38,"title":39,"description":40,"date":41,"categories":40,"tags":42,"draft":36},"\u002Fposts\u002Fmathematical-foundations-of-llms-1","LLM的数学基础（1）：","","2026-09-19 22:59:55",null,{"path":44,"title":45,"description":40,"date":46,"categories":40,"tags":42,"draft":36},"\u002Fposts\u002Ftowards-maximize-serving-capatity","如何在 offline bench 里最大化 LLM serving system 的吞吐","2026-09-19 05:45:05",{"path":48,"title":49,"description":50,"date":51,"categories":52,"tags":53,"draft":15},"\u002Fposts\u002Fllm-memory-compute-tradeoff","LLM 的 Memory-Compute Trade-off：从 MHA 到 MLA、Sparse 与 Linear Attention","ds4.1f 上线了。它的架构 novel 在哪，便宜和快又是从哪省出来的。","2026-09-11 12:00:00","技术杂谈",[54,55,56,57,58],"LLMs","Attention","MLA","DeepSeek","长上下文",{"path":60,"title":61,"description":62,"date":63,"categories":52,"tags":64,"draft":36},"\u002Fposts\u002Fmarkdown-extensions","Markdown 扩展语法速查","本站 Markdown 支持的扩展语法演示与速查：admonition 提示块、GitHub 仓库卡片、spoiler 遮罩、ABC 乐谱、音频播放器、浏览器可运行代码块。","2026-09-01 15:30:00",[65],"博客",{"path":67,"title":68,"description":69,"date":70,"categories":31,"tags":71,"draft":15},"\u002Fposts\u002Fwhy-we-still-need-to-write-a-blog","为什么还需要手写博客？","AI这么强，还要古法写博客吗？","2026-09-01 13:17:53",[65],{"path":73,"title":74,"description":40,"date":75,"categories":40,"tags":42,"draft":36},"\u002Fposts\u002Fllm-serving-1","LLM Serving（一）:以在H200上部署GLM-5.2为例","2026-07-21 06:15:44",{"path":77,"title":78,"description":40,"date":79,"categories":80,"tags":42,"draft":15},"\u002Fposts\u002Finteresting-music-rhythm","有趣的音乐节奏","2026-07-16 12:47:24","音乐",{"path":82,"title":83,"description":84,"date":85,"categories":9,"tags":86,"draft":15},"\u002Fposts\u002Ftowards-monosemanticity","LLM 可解释性（一）：SAE与单语义","读 Anthropic《Towards Monosemanticity》。从难以解释的神经元出发, 讲清superposition为什么让单个神经元混成一团, SAE怎么把多语义的激活分解成单语义特征。","2026-07-12 22:00:00",[54,87,88],"可解释性","SAE",{"path":90,"title":91,"description":92,"date":93,"categories":31,"tags":94,"draft":36},"\u002Fposts\u002Fpiketty-ai-china-k-shaped","用皮凯蒂的镜头看 AI 与中国 K 型社会","一次凌晨的对话整理。从 r>g 出发，看 AI 时代的资本分配、中国的世袭化趋势、体制内的二元劳动力市场，以及一个普通 00 后在一线打拼的真实牌面。","2026-06-24 03:00:00",[34,95,33],"经济",{"path":97,"title":98,"description":40,"date":99,"categories":40,"tags":100,"draft":36},"\u002Fposts\u002Fclaude-code-compact-behavior","cluade code compact 行为","2026-06-23 20:17:42",[101,102,103],"Agent","AI Agent","Claude Code",{"path":105,"title":106,"description":40,"date":107,"categories":9,"tags":108,"draft":15},"\u002Fposts\u002Fclaude-code-settings","Claude Code 配置分享","2026-06-23 14:22:12",[54,102,101,103],{"path":110,"title":111,"description":112,"date":113,"categories":52,"tags":114,"draft":15},"\u002Fposts\u002Fmy-thoughts-on-the-ai-industry","简单聊聊对 AI Agent 行业的理解","写这篇文章的初衷，是我发觉逐渐忘了前两个月的 AI\u002FLLMs 是什么样的能力和情况了。意识中似乎某些在突然加速，但理性和直觉告诉我这不对，作为行内人，我知道这是一个渐进的、累积的过程。我想自己回忆一下，Agent的发展。","2026-06-20 13:14:25",[54,102,101],{"path":116,"title":117,"description":40,"date":118,"categories":9,"tags":119,"draft":15},"\u002Fposts\u002Frotational-positional-encoding","RoPE","2026-06-17 18:31:40",[54,120,117],"Transformer",{"path":122,"title":123,"description":40,"date":124,"categories":9,"tags":125,"draft":15},"\u002Fposts\u002Fintroduction-to-large-language-model","大语言模型入门基础","2026-06-16 22:36:11",[120,54],{"path":127,"title":128,"description":40,"date":129,"categories":52,"tags":130,"draft":15},"\u002Fposts\u002Fthought-about-coding-agent","关于 Coding Agent 使用的遐思","2026-06-03 17:54:20",[101],{"path":132,"title":133,"description":134,"date":135,"categories":9,"tags":136,"draft":15},"\u002Fposts\u002Fabout-ssh-usage","关于SSH的使用——客户端篇","介绍SSH客户端的用法。","2025-06-17 02:44:57",[137],"工具",{"path":139,"title":140,"description":141,"date":142,"categories":52,"tags":143,"draft":15},"\u002Fposts\u002Fopensourcesoftwarestarttutorial","如何开始一个开源项目？","浅谈我对开源项目的认知。","2025-04-17 18:00:00",[144],"开源",{"path":146,"title":147,"description":148,"date":149,"categories":150,"tags":151,"draft":15},"\u002Fposts\u002Fmotivation","渴求动力","有意识的生活并主动地行动会使得生活更美好。","2025-03-05 00:44:57","生活-情感",[34],{"path":153,"title":154,"description":155,"date":156,"categories":150,"tags":157,"draft":15},"\u002Fposts\u002F2024summary","2024年终总结","啊，痛彻的时光；啊，悲怆的人！","2025-01-01 01:00:00",[34],{"path":159,"title":160,"description":161,"date":162,"categories":9,"tags":163,"draft":15},"\u002Fposts\u002Fvps-crash-startup","快速搭建个人VPS","如何快速搭建个人VPS。","2024-12-31 19:00:00",[164],"VPS",{"path":166,"title":167,"description":168,"date":169,"categories":9,"tags":170,"draft":15},"\u002Fposts\u002Fdata-label-software","AI数据（自动）标注软件概述","对市面上目前的AI数据（自动）标注软件进行调研。","2024-12-20 14:00:00",[171,172],"数据标注","深度学习",{"path":174,"title":175,"description":176,"date":177,"categories":150,"tags":178,"draft":15},"\u002Fposts\u002Fself-destruction","生活的崩坏与自我毁灭","是什么在牵引着我？我的行为，我的思想？我的感受？我所处的地方？它们在何处？","2024-12-19 12:00:00",[34],{"path":180,"title":181,"description":182,"date":183,"categories":9,"tags":184,"draft":15},"\u002Fposts\u002Fcs194-196_llmagents","LLM Agents 课程笔记","UCB CS294\u002F194-196 Large Language Model Agents 的课程笔记。","2024-11-30 20:00:00",[101],{"path":186,"title":187,"description":188,"date":189,"categories":150,"tags":190,"draft":15},"\u002Fposts\u002Fwhatsyourworth","价值、位置","你们的生存究竟有何价值？如果毫无价值，你们究竟为何存在？","2024-11-29 02:00:00",[34],{"path":192,"title":193,"description":194,"date":195,"categories":150,"tags":196,"draft":15},"\u002Fposts\u002Fblogrelifever2","重启博客v2","忙里偷闲学习前端，还是决定结合想法手搓一个自己的博客。","2024-11-25 00:00:00",[65],{"path":198,"title":199,"description":200,"date":201,"categories":150,"tags":42,"draft":15},"\u002Fposts\u002Fblogrelife","重启博客","在我的表达欲与现实所需(问题解决记录)驱动下，我决定重写博客。","2024-10-21 20:44:57",{"id":203,"title":6,"body":204,"categories":9,"date":19,"description":18,"draft":15,"extension":2114,"meta":2115,"navigation":36,"path":17,"seo":2116,"stem":2118,"tags":2119,"__hash__":2120},"posts\u002Fposts\u002Fspinning-up-rl-intro-2.md",{"type":205,"value":206,"toc":2094},"minimark",[207,235,238,242,249,255,258,271,275,278,287,290,302,305,308,325,329,332,610,789,807,1016,1242,1472,1475,1552,1556,1636,1640,1649,1667,1670,1673,1677,1686,1689,1699,1702,1803,1806,1822,1825,1828,1837,1855,1858,1861,1864,1874,1877,1884,1913,1918],[208,209,211],"note",{"title":210},"译者说明",[212,213,214,215,222,223,228,229,234],"p",{},"本文是 OpenAI ",[216,217,221],"a",{"href":218,"rel":219},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002F",[220],"nofollow","Spinning Up in Deep RL"," 教程第二部分 ",[216,224,227],{"href":225,"rel":226},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Frl_intro2.html",[220],"Part 2: Kinds of RL Algorithms"," 的全文翻译。原仓库采用 ",[216,230,233],{"href":231,"rel":232},"https:\u002F\u002Fgithub.com\u002Fopenai\u002Fspinningup\u002Fblob\u002Fmaster\u002FLICENSE",[220],"MIT 协议","。分类图保留官方英文原图，正文保留算法缩写与外链；术语首次出现时附英文原词，后文沿用同一译名。",[212,236,237],{},"术语和记号打好基础后，接下来可以看更丰富的内容：现代 RL 算法的整体版图，以及算法设计中各种取舍。",[239,240,241],"h2",{"id":241},"强化学习算法的分类",[212,243,244],{},[245,246],"img",{"alt":247,"src":248},"现代强化学习算法的一种分类图（官方英文原图）","https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002F_images\u002Frl_algorithms_9_15.svg",[212,250,251],{},[252,253,254],"em",{},"图 1：现代 RL 算法的一种实用分类，并未穷尽所有方法。算法引用见文末。",[212,256,257],{},"先说明一点：现代 RL 算法的模块化特征很难用树形结构表示，因此要画出准确且包罗万象的算法分类图相当困难。为了让分类图能放在一页内，并使这篇导论便于阅读，我们省略了不少进阶主题，例如探索、迁移学习和元学习。本文的目标是：",[259,260,261,265,268],"ul",{},[262,263,264],"li",{},"突出深度 RL 算法中最基础的设计选择：要学习什么，以及怎样学习；",[262,266,267],{},"展示这些选择各自的取舍；",[262,269,270],{},"将几种重要的现代算法放进这些选择构成的脉络中。",[272,273,274],"h3",{"id":274},"无模型与基于模型的强化学习",[212,276,277],{},"强化学习算法面临的一个重要分岔是：智能体能否访问环境模型，或者能否学到一个环境模型。这里的环境模型指预测状态转移和奖励的函数。",[212,279,280,281,286],{},"使用模型的主要好处，是智能体可以向前规划：设想若干种选择分别会带来什么结果，再明确比较这些选择。随后，智能体还可以把规划所得提炼成一个学到的策略。一个著名例子是 ",[216,282,285],{"href":283,"rel":284},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1712.01815",[220],"AlphaZero","。模型方法若能奏效，相较于不使用模型的方法，样本效率（sample efficiency）可能会大幅提高。",[212,288,289],{},"主要困难在于，智能体通常拿不到环境的真实模型。此时若想使用模型，就只能从经验中学习它；这会带来一系列挑战。最突出的一点是，智能体可能利用模型中的偏差，在学到的模型里表现良好，到了真实环境中却表现欠佳，甚至极差。模型学习本身就很难；即使投入大量时间和算力，也可能得不到相应收益。",[212,291,292,293,297,298,301],{},"使用环境模型的算法称为",[294,295,296],"strong",{},"基于模型的方法（model-based methods）","，不使用环境模型的算法称为",[294,299,300],{},"无模型方法（model-free methods）","。无模型方法放弃了借助模型提高样本效率的潜在收益，但通常更容易实现和调参。截至这篇导论写作时（2018 年 9 月），无模型方法更受欢迎，也经过了更多开发和测试。",[272,303,304],{"id":304},"要学习什么",[212,306,307],{},"另一个关键分岔是算法要学习什么。常见的学习对象包括：",[259,309,310,313,316,319,322],{},[262,311,312],{},"策略，可以是随机型策略，也可以是确定型策略；",[262,314,315],{},"动作价值函数（Q 函数）；",[262,317,318],{},"价值函数；",[262,320,321],{},"环境模型；",[262,323,324],{},"或以上对象的组合。",[326,327,328],"h4",{"id":328},"无模型强化学习要学习什么",[212,330,331],{},"无模型 RL 中，表征和训练智能体主要有两种方法。",[212,333,334,337,338,483,484,514,515,605,606,609],{},[294,335,336],{},"策略优化（policy optimization）","。这类方法显式地用策略 ",[339,340,343,389],"span",{"className":341},[342],"katex",[339,344,347],{"className":345},[346],"katex-mathml",[348,349,351],"math",{"xmlns":350},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[352,353,354,384],"semantics",{},[355,356,357,367,372,374,378,381],"mrow",{},[358,359,360,364],"msub",{},[361,362,363],"mi",{},"π",[361,365,366],{},"θ",[368,369,371],"mo",{"stretchy":370},"false","(",[361,373,216],{},[361,375,377],{"mathvariant":376},"normal","∣",[361,379,380],{},"s",[368,382,383],{"stretchy":370},")",[385,386,388],"annotation",{"encoding":387},"application\u002Fx-tex","\\pi_{\\theta}(a|s)",[339,390,394],{"className":391,"ariaHidden":393},[392],"katex-html","true",[339,395,398,403,466,470,473,476,479],{"className":396},[397],"base",[339,399],{"className":400,"style":402},[401],"strut","height:1em;vertical-align:-0.25em;",[339,404,407,412],{"className":405},[406],"mord",[339,408,363],{"className":409,"style":411},[406,410],"mathnormal","margin-right:0.0359em;",[339,413,416],{"className":414},[415],"msupsub",[339,417,421,457],{"className":418},[419,420],"vlist-t","vlist-t2",[339,422,425,452],{"className":423},[424],"vlist-r",[339,426,430],{"className":427,"style":429},[428],"vlist","height:0.3361em;",[339,431,433,438],{"style":432},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[339,434],{"className":435,"style":437},[436],"pstrut","height:2.7em;",[339,439,445],{"className":440},[441,442,443,444],"sizing","reset-size6","size3","mtight",[339,446,448],{"className":447},[406,444],[339,449,366],{"className":450,"style":451},[406,410,444],"margin-right:0.0278em;",[339,453,456],{"className":454},[455],"vlist-s","​",[339,458,460],{"className":459},[424],[339,461,464],{"className":462,"style":463},[428],"height:0.15em;",[339,465],{},[339,467,371],{"className":468},[469],"mopen",[339,471,216],{"className":472},[406,410],[339,474,377],{"className":475},[406],[339,477,380],{"className":478},[406,410],[339,480,383],{"className":481},[482],"mclose"," 表示智能体。它们通过两种方式优化参数 ",[339,485,487,501],{"className":486},[342],[339,488,490],{"className":489},[346],[348,491,492],{"xmlns":350},[352,493,494,498],{},[355,495,496],{},[361,497,366],{},[385,499,500],{"encoding":387},"\\theta",[339,502,504],{"className":503,"ariaHidden":393},[392],[339,505,507,511],{"className":506},[397],[339,508],{"className":509,"style":510},[401],"height:0.6944em;",[339,512,366],{"className":513,"style":451},[406,410],"：直接对性能目标 ",[339,516,518,543],{"className":517},[342],[339,519,521],{"className":520},[346],[348,522,523],{"xmlns":350},[352,524,525,540],{},[355,526,527,530,532,538],{},[361,528,529],{},"J",[368,531,371],{"stretchy":370},[358,533,534,536],{},[361,535,363],{},[361,537,366],{},[368,539,383],{"stretchy":370},[385,541,542],{"encoding":387},"J(\\pi_{\\theta})",[339,544,546],{"className":545,"ariaHidden":393},[392],[339,547,549,552,556,559,602],{"className":548},[397],[339,550],{"className":551,"style":402},[401],[339,553,529],{"className":554,"style":555},[406,410],"margin-right:0.0962em;",[339,557,371],{"className":558},[469],[339,560,562,565],{"className":561},[406],[339,563,363],{"className":564,"style":411},[406,410],[339,566,568],{"className":567},[415],[339,569,571,594],{"className":570},[419,420],[339,572,574,591],{"className":573},[424],[339,575,577],{"className":576,"style":429},[428],[339,578,579,582],{"style":432},[339,580],{"className":581,"style":437},[436],[339,583,585],{"className":584},[441,442,443,444],[339,586,588],{"className":587},[406,444],[339,589,366],{"className":590,"style":451},[406,410,444],[339,592,456],{"className":593},[455],[339,595,597],{"className":596},[424],[339,598,600],{"className":599,"style":463},[428],[339,601],{},[339,603,383],{"className":604},[482]," 做梯度上升，或者间接地最大化该目标的局部代理（surrogate）。优化过程几乎总是采用",[294,607,608],{},"同策略（on-policy）"," 方式：每次更新只使用智能体按照当前最新策略与环境交互时收集的数据。",[212,611,612,613,707,708,788],{},"策略优化通常还会学习一个函数 ",[339,614,616,642],{"className":615},[342],[339,617,619],{"className":618},[346],[348,620,621],{"xmlns":350},[352,622,623,639],{},[355,624,625,633,635,637],{},[358,626,627,630],{},[361,628,629],{},"V",[361,631,632],{},"ϕ",[368,634,371],{"stretchy":370},[361,636,380],{},[368,638,383],{"stretchy":370},[385,640,641],{"encoding":387},"V_{\\phi}(s)",[339,643,645],{"className":644,"ariaHidden":393},[392],[339,646,648,652,698,701,704],{"className":647},[397],[339,649],{"className":650,"style":651},[401],"height:1.0361em;vertical-align:-0.2861em;",[339,653,655,659],{"className":654},[406],[339,656,629],{"className":657,"style":658},[406,410],"margin-right:0.2222em;",[339,660,662],{"className":661},[415],[339,663,665,689],{"className":664},[419,420],[339,666,668,686],{"className":667},[424],[339,669,671],{"className":670,"style":429},[428],[339,672,674,677],{"style":673},"top:-2.55em;margin-left:-0.2222em;margin-right:0.05em;",[339,675],{"className":676,"style":437},[436],[339,678,680],{"className":679},[441,442,443,444],[339,681,683],{"className":682},[406,444],[339,684,632],{"className":685},[406,410,444],[339,687,456],{"className":688},[455],[339,690,692],{"className":691},[424],[339,693,696],{"className":694,"style":695},[428],"height:0.2861em;",[339,697],{},[339,699,371],{"className":700},[469],[339,702,380],{"className":703},[406,410],[339,705,383],{"className":706},[482],"，用来近似该策略下的价值函数 ",[339,709,711,736],{"className":710},[342],[339,712,714],{"className":713},[346],[348,715,716],{"xmlns":350},[352,717,718,733],{},[355,719,720,727,729,731],{},[721,722,723,725],"msup",{},[361,724,629],{},[361,726,363],{},[368,728,371],{"stretchy":370},[361,730,380],{},[368,732,383],{"stretchy":370},[385,734,735],{"encoding":387},"V^{\\pi}(s)",[339,737,739],{"className":738,"ariaHidden":393},[392],[339,740,742,745,779,782,785],{"className":741},[397],[339,743],{"className":744,"style":402},[401],[339,746,748,751],{"className":747},[406],[339,749,629],{"className":750,"style":658},[406,410],[339,752,754],{"className":753},[415],[339,755,757],{"className":756},[419],[339,758,760],{"className":759},[424],[339,761,764],{"className":762,"style":763},[428],"height:0.6644em;",[339,765,767,770],{"style":766},"top:-3.063em;margin-right:0.05em;",[339,768],{"className":769,"style":437},[436],[339,771,773],{"className":772},[441,442,443,444],[339,774,776],{"className":775},[406,444],[339,777,363],{"className":778,"style":411},[406,410,444],[339,780,371],{"className":781},[469],[339,783,380],{"className":784},[406,410],[339,786,383],{"className":787},[482],"，并据此计算如何更新策略。策略优化方法的例子包括：",[259,790,791,799],{},[262,792,793,798],{},[216,794,797],{"href":795,"rel":796},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1602.01783",[220],"A2C \u002F A3C","：通过梯度上升直接最大化性能；",[262,800,801,806],{},[216,802,805],{"href":803,"rel":804},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1707.06347",[220],"PPO","：通过最大化一个代理目标函数，间接推动性能目标上升。这个代理目标会保守估计这次策略更新可能引起的性能目标变化。",[212,808,809,812,813,920,921,1011,1012,1015],{},[294,810,811],{},"Q-learning","。这类方法学习一个函数 ",[339,814,816,846],{"className":815},[342],[339,817,819],{"className":818},[346],[348,820,821],{"xmlns":350},[352,822,823,843],{},[355,824,825,832,834,836,839,841],{},[358,826,827,830],{},[361,828,829],{},"Q",[361,831,366],{},[368,833,371],{"stretchy":370},[361,835,380],{},[368,837,838],{"separator":393},",",[361,840,216],{},[368,842,383],{"stretchy":370},[385,844,845],{"encoding":387},"Q_{\\theta}(s,a)",[339,847,849],{"className":848,"ariaHidden":393},[392],[339,850,852,855,899,902,905,909,914,917],{"className":851},[397],[339,853],{"className":854,"style":402},[401],[339,856,858,861],{"className":857},[406],[339,859,829],{"className":860},[406,410],[339,862,864],{"className":863},[415],[339,865,867,891],{"className":866},[419,420],[339,868,870,888],{"className":869},[424],[339,871,873],{"className":872,"style":429},[428],[339,874,876,879],{"style":875},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[339,877],{"className":878,"style":437},[436],[339,880,882],{"className":881},[441,442,443,444],[339,883,885],{"className":884},[406,444],[339,886,366],{"className":887,"style":451},[406,410,444],[339,889,456],{"className":890},[455],[339,892,894],{"className":893},[424],[339,895,897],{"className":896,"style":463},[428],[339,898],{},[339,900,371],{"className":901},[469],[339,903,380],{"className":904},[406,410],[339,906,838],{"className":907},[908],"mpunct",[339,910],{"className":911,"style":913},[912],"mspace","margin-right:0.1667em;",[339,915,216],{"className":916},[406,410],[339,918,383],{"className":919},[482],"，用来近似最优动作价值函数 ",[339,922,924,953],{"className":923},[342],[339,925,927],{"className":926},[346],[348,928,929],{"xmlns":350},[352,930,931,950],{},[355,932,933,940,942,944,946,948],{},[721,934,935,937],{},[361,936,829],{},[368,938,939],{},"∗",[368,941,371],{"stretchy":370},[361,943,380],{},[368,945,838],{"separator":393},[361,947,216],{},[368,949,383],{"stretchy":370},[385,951,952],{"encoding":387},"Q^*(s,a)",[339,954,956],{"className":955,"ariaHidden":393},[392],[339,957,959,962,993,996,999,1002,1005,1008],{"className":958},[397],[339,960],{"className":961,"style":402},[401],[339,963,965,968],{"className":964},[406],[339,966,829],{"className":967},[406,410],[339,969,971],{"className":970},[415],[339,972,974],{"className":973},[419],[339,975,977],{"className":976},[424],[339,978,981],{"className":979,"style":980},[428],"height:0.6887em;",[339,982,983,986],{"style":766},[339,984],{"className":985,"style":437},[436],[339,987,989],{"className":988},[441,442,443,444],[339,990,939],{"className":991},[992,444],"mbin",[339,994,371],{"className":995},[469],[339,997,380],{"className":998},[406,410],[339,1000,838],{"className":1001},[908],[339,1003],{"className":1004,"style":913},[912],[339,1006,216],{"className":1007},[406,410],[339,1009,383],{"className":1010},[482],"。它们通常使用基于贝尔曼方程的目标函数。优化过程几乎总是采用",[294,1013,1014],{},"异策略（off-policy）"," 方式：每次更新可以使用训练过程中任意时刻收集的数据，而不受智能体采集这些数据时采用何种探索策略的限制。",[212,1017,1018,1019,1105,1106,1180,1181,1241],{},"最优策略会选择使 ",[339,1020,1022,1049],{"className":1021},[342],[339,1023,1025],{"className":1024},[346],[348,1026,1027],{"xmlns":350},[352,1028,1029,1047],{},[355,1030,1031,1037,1039,1041,1043,1045],{},[721,1032,1033,1035],{},[361,1034,829],{},[368,1036,939],{},[368,1038,371],{"stretchy":370},[361,1040,380],{},[368,1042,838],{"separator":393},[361,1044,216],{},[368,1046,383],{"stretchy":370},[385,1048,952],{"encoding":387},[339,1050,1052],{"className":1051,"ariaHidden":393},[392],[339,1053,1055,1058,1087,1090,1093,1096,1099,1102],{"className":1054},[397],[339,1056],{"className":1057,"style":402},[401],[339,1059,1061,1064],{"className":1060},[406],[339,1062,829],{"className":1063},[406,410],[339,1065,1067],{"className":1066},[415],[339,1068,1070],{"className":1069},[419],[339,1071,1073],{"className":1072},[424],[339,1074,1076],{"className":1075,"style":980},[428],[339,1077,1078,1081],{"style":766},[339,1079],{"className":1080,"style":437},[436],[339,1082,1084],{"className":1083},[441,442,443,444],[339,1085,939],{"className":1086},[992,444],[339,1088,371],{"className":1089},[469],[339,1091,380],{"className":1092},[406,410],[339,1094,838],{"className":1095},[908],[339,1097],{"className":1098,"style":913},[912],[339,1100,216],{"className":1101},[406,410],[339,1103,383],{"className":1104},[482]," 最大的动作；Q-learning 则用学得的 ",[339,1107,1109,1127],{"className":1108},[342],[339,1110,1112],{"className":1111},[346],[348,1113,1114],{"xmlns":350},[352,1115,1116,1124],{},[355,1117,1118],{},[358,1119,1120,1122],{},[361,1121,829],{},[361,1123,366],{},[385,1125,1126],{"encoding":387},"Q_{\\theta}",[339,1128,1130],{"className":1129,"ariaHidden":393},[392],[339,1131,1133,1137],{"className":1132},[397],[339,1134],{"className":1135,"style":1136},[401],"height:0.8778em;vertical-align:-0.1944em;",[339,1138,1140,1143],{"className":1139},[406],[339,1141,829],{"className":1142},[406,410],[339,1144,1146],{"className":1145},[415],[339,1147,1149,1172],{"className":1148},[419,420],[339,1150,1152,1169],{"className":1151},[424],[339,1153,1155],{"className":1154,"style":429},[428],[339,1156,1157,1160],{"style":875},[339,1158],{"className":1159,"style":437},[436],[339,1161,1163],{"className":1162},[441,442,443,444],[339,1164,1166],{"className":1165},[406,444],[339,1167,366],{"className":1168,"style":451},[406,410,444],[339,1170,456],{"className":1171},[455],[339,1173,1175],{"className":1174},[424],[339,1176,1178],{"className":1177,"style":463},[428],[339,1179],{}," 近似 ",[339,1182,1184,1202],{"className":1183},[342],[339,1185,1187],{"className":1186},[346],[348,1188,1189],{"xmlns":350},[352,1190,1191,1199],{},[355,1192,1193],{},[721,1194,1195,1197],{},[361,1196,829],{},[368,1198,939],{},[385,1200,1201],{"encoding":387},"Q^*",[339,1203,1205],{"className":1204,"ariaHidden":393},[392],[339,1206,1208,1212],{"className":1207},[397],[339,1209],{"className":1210,"style":1211},[401],"height:0.8831em;vertical-align:-0.1944em;",[339,1213,1215,1218],{"className":1214},[406],[339,1216,829],{"className":1217},[406,410],[339,1219,1221],{"className":1220},[415],[339,1222,1224],{"className":1223},[419],[339,1225,1227],{"className":1226},[424],[339,1228,1230],{"className":1229,"style":980},[428],[339,1231,1232,1235],{"style":766},[339,1233],{"className":1234,"style":437},[436],[339,1236,1238],{"className":1237},[441,442,443,444],[339,1239,939],{"className":1240},[992,444],"，据此选取动作：",[339,1243,1246],{"className":1244},[1245],"katex-display",[339,1247,1249,1307],{"className":1248},[342],[339,1250,1252],{"className":1251},[346],[348,1253,1255],{"xmlns":350,"display":1254},"block",[352,1256,1257,1304],{},[355,1258,1259,1261,1263,1265,1267,1270,1273,1276,1288,1294,1296,1298,1300,1302],{},[361,1260,216],{},[368,1262,371],{"stretchy":370},[361,1264,380],{},[368,1266,383],{"stretchy":370},[368,1268,1269],{},"=",[361,1271,1272],{},"arg",[368,1274,1275],{},"⁡",[1277,1278,1279,1286],"munder",{},[355,1280,1281,1284],{},[361,1282,1283],{},"max",[368,1285,1275],{},[361,1287,216],{},[358,1289,1290,1292],{},[361,1291,829],{},[361,1293,366],{},[368,1295,371],{"stretchy":370},[361,1297,380],{},[368,1299,838],{"separator":393},[361,1301,216],{},[368,1303,383],{"stretchy":370},[385,1305,1306],{"encoding":387},"a(s) = \\arg\\max_a Q_{\\theta}(s,a)",[339,1308,1310,1339],{"className":1309,"ariaHidden":393},[392],[339,1311,1313,1316,1319,1322,1325,1328,1332,1336],{"className":1312},[397],[339,1314],{"className":1315,"style":402},[401],[339,1317,216],{"className":1318},[406,410],[339,1320,371],{"className":1321},[469],[339,1323,380],{"className":1324},[406,410],[339,1326,383],{"className":1327},[482],[339,1329],{"className":1330,"style":1331},[912],"margin-right:0.2778em;",[339,1333,1269],{"className":1334},[1335],"mrel",[339,1337],{"className":1338,"style":1331},[912],[339,1340,1342,1346,1355,1358,1408,1411,1454,1457,1460,1463,1466,1469],{"className":1341},[397],[339,1343],{"className":1344,"style":1345},[401],"height:1.45em;vertical-align:-0.7em;",[339,1347,1350,1351],{"className":1348},[1349],"mop","ar",[339,1352,1354],{"style":1353},"margin-right:0.0139em;","g",[339,1356],{"className":1357,"style":913},[912],[339,1359,1362],{"className":1360},[1349,1361],"op-limits",[339,1363,1365,1399],{"className":1364},[419,420],[339,1366,1368,1396],{"className":1367},[424],[339,1369,1372,1385],{"className":1370,"style":1371},[428],"height:0.4306em;",[339,1373,1375,1379],{"style":1374},"top:-2.4em;margin-left:0em;",[339,1376],{"className":1377,"style":1378},[436],"height:3em;",[339,1380,1382],{"className":1381},[441,442,443,444],[339,1383,216],{"className":1384},[406,410,444],[339,1386,1388,1391],{"style":1387},"top:-3em;",[339,1389],{"className":1390,"style":1378},[436],[339,1392,1393],{},[339,1394,1283],{"className":1395},[1349],[339,1397,456],{"className":1398},[455],[339,1400,1402],{"className":1401},[424],[339,1403,1406],{"className":1404,"style":1405},[428],"height:0.7em;",[339,1407],{},[339,1409],{"className":1410,"style":913},[912],[339,1412,1414,1417],{"className":1413},[406],[339,1415,829],{"className":1416},[406,410],[339,1418,1420],{"className":1419},[415],[339,1421,1423,1446],{"className":1422},[419,420],[339,1424,1426,1443],{"className":1425},[424],[339,1427,1429],{"className":1428,"style":429},[428],[339,1430,1431,1434],{"style":875},[339,1432],{"className":1433,"style":437},[436],[339,1435,1437],{"className":1436},[441,442,443,444],[339,1438,1440],{"className":1439},[406,444],[339,1441,366],{"className":1442,"style":451},[406,410,444],[339,1444,456],{"className":1445},[455],[339,1447,1449],{"className":1448},[424],[339,1450,1452],{"className":1451,"style":463},[428],[339,1453],{},[339,1455,371],{"className":1456},[469],[339,1458,380],{"className":1459},[406,410],[339,1461,838],{"className":1462},[908],[339,1464],{"className":1465,"style":913},[912],[339,1467,216],{"className":1468},[406,410],[339,1470,383],{"className":1471},[482],[212,1473,1474],{},"Q-learning 方法的例子包括：",[259,1476,1477,1485],{},[262,1478,1479,1484],{},[216,1480,1483],{"href":1481,"rel":1482},"https:\u002F\u002Fwww.cs.toronto.edu\u002F~vmnih\u002Fdocs\u002Fdqn.pdf",[220],"DQN","：推动深度 RL 领域发展的经典方法；",[262,1486,1487,1492,1493,1551],{},[216,1488,1491],{"href":1489,"rel":1490},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1707.06887",[220],"C51","：一种学习回报分布的变体，其期望就是 ",[339,1494,1496,1513],{"className":1495},[342],[339,1497,1499],{"className":1498},[346],[348,1500,1501],{"xmlns":350},[352,1502,1503,1511],{},[355,1504,1505],{},[721,1506,1507,1509],{},[361,1508,829],{},[368,1510,939],{},[385,1512,1201],{"encoding":387},[339,1514,1516],{"className":1515,"ariaHidden":393},[392],[339,1517,1519,1522],{"className":1518},[397],[339,1520],{"className":1521,"style":1211},[401],[339,1523,1525,1528],{"className":1524},[406],[339,1526,829],{"className":1527},[406,410],[339,1529,1531],{"className":1530},[415],[339,1532,1534],{"className":1533},[419],[339,1535,1537],{"className":1536},[424],[339,1538,1540],{"className":1539,"style":980},[428],[339,1541,1542,1545],{"style":766},[339,1543],{"className":1544,"style":437},[436],[339,1546,1548],{"className":1547},[441,442,443,444],[339,1549,939],{"className":1550},[992,444],"。",[326,1553,1555],{"id":1554},"策略优化与-q-learning-的取舍","策略优化与 Q-learning 的取舍",[212,1557,1558,1559,1631,1632,1635],{},"策略优化方法的主要优点是目标明确：它直接优化我们真正关心的对象，因此往往稳定、可靠。相比之下，Q-learning 通过训练 ",[339,1560,1562,1579],{"className":1561},[342],[339,1563,1565],{"className":1564},[346],[348,1566,1567],{"xmlns":350},[352,1568,1569,1577],{},[355,1570,1571],{},[358,1572,1573,1575],{},[361,1574,829],{},[361,1576,366],{},[385,1578,1126],{"encoding":387},[339,1580,1582],{"className":1581,"ariaHidden":393},[392],[339,1583,1585,1588],{"className":1584},[397],[339,1586],{"className":1587,"style":1136},[401],[339,1589,1591,1594],{"className":1590},[406],[339,1592,829],{"className":1593},[406,410],[339,1595,1597],{"className":1596},[415],[339,1598,1600,1623],{"className":1599},[419,420],[339,1601,1603,1620],{"className":1602},[424],[339,1604,1606],{"className":1605,"style":429},[428],[339,1607,1608,1611],{"style":875},[339,1609],{"className":1610,"style":437},[436],[339,1612,1614],{"className":1613},[441,442,443,444],[339,1615,1617],{"className":1616},[406,444],[339,1618,366],{"className":1619,"style":451},[406,410,444],[339,1621,456],{"className":1622},[455],[339,1624,1626],{"className":1625},[424],[339,1627,1629],{"className":1628,"style":463},[428],[339,1630],{}," 满足自洽方程，间接推动智能体表现变好。这类学习存在多种失效模式，因此通常稳定性较差（相关讨论见文末参考资料 ",[339,1633,1634],{},"1","）。不过，在能够奏效时，Q-learning 的样本效率明显更高，因为它比策略优化更充分地复用数据。",[326,1637,1639],{"id":1638},"介于策略优化与-q-learning-之间的方法","介于策略优化与 Q-learning 之间的方法",[212,1641,1642,1643,1648],{},"策略优化与 Q-learning 并不互斥，在",[216,1644,1647],{"href":1645,"rel":1646},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1704.06440",[220],"某些条件下甚至可以等价","。处于两者之间的一系列算法，可以在两端方法的优缺点之间作出折中。例如：",[259,1650,1651,1659],{},[262,1652,1653,1658],{},[216,1654,1657],{"href":1655,"rel":1656},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1509.02971",[220],"DDPG"," 同时学习一个确定型策略和一个 Q 函数，并利用两者相互改进；",[262,1660,1661,1666],{},[216,1662,1665],{"href":1663,"rel":1664},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1801.01290",[220],"SAC"," 使用随机策略、熵正则化和其他技巧来稳定学习，在标准基准上的得分高于 DDPG。",[272,1668,1669],{"id":1669},"基于模型的强化学习要学习什么",[212,1671,1672],{},"与无模型 RL 不同，基于模型的方法很难归纳成少数几类边界清晰的算法；模型的使用方式有许多彼此独立的维度。下面只举几个例子，远未穷尽。每种方法使用的模型都可能是直接给定的，也可能是从经验中学到的。",[326,1674,1676],{"id":1675},"背景纯规划","背景：纯规划",[212,1678,1679,1680,1685],{},"最基础的做法是不显式表示策略，而是使用 ",[216,1681,1684],{"href":1682,"rel":1683},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FModel_predictive_control",[220],"模型预测控制（model-predictive control，MPC）"," 这样的纯规划方法来选择动作。MPC 中，智能体每次观测环境时，都会根据模型计算一个最优计划；计划规定从当前时刻起，在一个固定时间窗口内要采取哪些动作。",[212,1687,1688],{},"规划算法也可以借助学到的价值函数，考虑规划视界之外的未来奖励。随后，智能体只执行计划中的第一个动作，并立即丢弃计划的其余部分。每次准备与环境交互时，智能体都会重新计算计划，以免继续执行旧计划时规划视界已短于预期。",[259,1690,1691],{},[262,1692,1693,1698],{},[216,1694,1697],{"href":1695,"rel":1696},"https:\u002F\u002Fsites.google.com\u002Fview\u002Fmbmf",[220],"MBMF"," 探索了在深度 RL 的标准基准任务中，使用学到的环境模型进行 MPC 的做法。",[326,1700,1701],{"id":1701},"专家迭代",[212,1703,1704,1705,1802],{},"在纯规划的基础上，一个直接的扩展是显式表示并学习策略 ",[339,1706,1708,1735],{"className":1707},[342],[339,1709,1711],{"className":1710},[346],[348,1712,1713],{"xmlns":350},[352,1714,1715,1733],{},[355,1716,1717,1723,1725,1727,1729,1731],{},[358,1718,1719,1721],{},[361,1720,363],{},[361,1722,366],{},[368,1724,371],{"stretchy":370},[361,1726,216],{},[361,1728,377],{"mathvariant":376},[361,1730,380],{},[368,1732,383],{"stretchy":370},[385,1734,388],{"encoding":387},[339,1736,1738],{"className":1737,"ariaHidden":393},[392],[339,1739,1741,1744,1787,1790,1793,1796,1799],{"className":1740},[397],[339,1742],{"className":1743,"style":402},[401],[339,1745,1747,1750],{"className":1746},[406],[339,1748,363],{"className":1749,"style":411},[406,410],[339,1751,1753],{"className":1752},[415],[339,1754,1756,1779],{"className":1755},[419,420],[339,1757,1759,1776],{"className":1758},[424],[339,1760,1762],{"className":1761,"style":429},[428],[339,1763,1764,1767],{"style":432},[339,1765],{"className":1766,"style":437},[436],[339,1768,1770],{"className":1769},[441,442,443,444],[339,1771,1773],{"className":1772},[406,444],[339,1774,366],{"className":1775,"style":451},[406,410,444],[339,1777,456],{"className":1778},[455],[339,1780,1782],{"className":1781},[424],[339,1783,1785],{"className":1784,"style":463},[428],[339,1786],{},[339,1788,371],{"className":1789},[469],[339,1791,216],{"className":1792},[406,410],[339,1794,377],{"className":1795},[406],[339,1797,380],{"className":1798},[406,410],[339,1800,383],{"className":1801},[482],"。智能体在模型中使用规划算法（例如蒙特卡洛树搜索）来规划，并从当前策略中采样候选动作。规划算法给出的动作比策略单独给出的动作更好，因此，规划算法可以充当相对于当前策略的“专家”。",[212,1804,1805],{},"之后，智能体更新策略，使它更倾向于给出与规划算法相似的动作。",[259,1807,1808,1816],{},[262,1809,1810,1815],{},[216,1811,1814],{"href":1812,"rel":1813},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1705.08439",[220],"ExIt"," 使用这种方法训练深度神经网络来玩 Hex。",[262,1817,1818,1821],{},[216,1819,285],{"href":283,"rel":1820},[220]," 也是这种方法的例子。",[326,1823,1824],{"id":1824},"用模型生成的经验增强无模型方法",[212,1826,1827],{},"使用无模型 RL 算法训练策略或 Q 函数，同时在更新智能体时采取以下做法之一：",[1829,1830,1831,1834],"ol",{},[262,1832,1833],{},"用模型生成的虚拟经验补充真实经验；",[262,1835,1836],{},"只使用模型生成的虚拟经验来更新智能体。",[259,1838,1839,1847],{},[262,1840,1841,1846],{},[216,1842,1845],{"href":1843,"rel":1844},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1803.00101",[220],"MBVE"," 展示了如何用虚拟经验补充真实经验。",[262,1848,1849,1854],{},[216,1850,1853],{"href":1851,"rel":1852},"https:\u002F\u002Fworldmodels.github.io\u002F",[220],"World Models"," 展示了如何只用虚拟经验训练智能体，并把这种方式称作“在梦境中训练”（training in the dream）。",[326,1856,1857],{"id":1857},"在策略中嵌入规划过程",[212,1859,1860],{},"另一种方法是把规划过程直接嵌入策略，作为策略调用的一个子程序；完整计划作为辅助信息提供给策略。再用任意标准的无模型算法训练策略。这里的关键是，策略可以学会选择在什么时候、以什么方式使用规划结果。",[212,1862,1863],{},"这样可以减轻模型偏差的影响：如果模型在某些状态下给出的规划不可靠，策略可以学会忽略它。",[259,1865,1866],{},[262,1867,1868,1873],{},[216,1869,1872],{"href":1870,"rel":1871},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1707.06203",[220],"I2A"," 是让智能体具备这类“想象”能力的一种方法。",[239,1875,1876],{"id":1876},"参考资料",[212,1878,1879],{},[294,1880,1881,1883],{},[339,1882,1634],{}," Q-learning 方法为何会失效",[259,1885,1886,1895,1904],{},[262,1887,1888,1889,1894],{},"Tsitsiklis 和 Van Roy 的",[216,1890,1893],{"href":1891,"rel":1892},"https:\u002F\u002Fweb.mit.edu\u002Fjnt\u002Fwww\u002FPapers\u002FJ063-97-bvr-td.pdf",[220],"经典论文","；",[262,1896,1897,1898,1903],{},"Szepesvári 的",[216,1899,1902],{"href":1900,"rel":1901},"https:\u002F\u002Fsites.ualberta.ca\u002F~szepesva\u002Fpapers\u002FRLAlgsInMDPs.pdf",[220],"综述","，第 4.3.2 节；",[262,1905,1906,1907,1912],{},"Sutton 与 Barto 的",[216,1908,1911],{"href":1909,"rel":1910},"http:\u002F\u002Fincompleteideas.net\u002Fbook\u002Fthe-book-2nd.html",[220],"《强化学习：导论》","第 11 章，尤其是第 11.3 节关于“致命三元组”（deadly triad）的讨论：函数逼近、自举更新和异策略数据共同导致价值学习算法不稳定。",[212,1914,1915],{},[294,1916,1917],{},"分类图中的算法引用",[259,1919,1920,1932,1943,1956,1967,1980,1991,2002,2013,2026,2039,2050,2061,2072,2083],{},[262,1921,1922,1931],{},[294,1923,1924,1927,1928],{},[339,1925,1926],{},"2"," ",[216,1929,797],{"href":795,"rel":1930},[220],"（Asynchronous Advantage Actor-Critic）：Mnih 等，2016。",[262,1933,1934,1942],{},[294,1935,1936,1927,1939],{},[339,1937,1938],{},"3",[216,1940,805],{"href":803,"rel":1941},[220],"（Proximal Policy Optimization）：Schulman 等，2017。",[262,1944,1945,1955],{},[294,1946,1947,1927,1950],{},[339,1948,1949],{},"4",[216,1951,1954],{"href":1952,"rel":1953},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1502.05477",[220],"TRPO","（Trust Region Policy Optimization）：Schulman 等，2015。",[262,1957,1958,1966],{},[294,1959,1960,1927,1963],{},[339,1961,1962],{},"5",[216,1964,1657],{"href":1655,"rel":1965},[220],"（Deep Deterministic Policy Gradient）：Lillicrap 等，2015。",[262,1968,1969,1979],{},[294,1970,1971,1927,1974],{},[339,1972,1973],{},"6",[216,1975,1978],{"href":1976,"rel":1977},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1802.09477",[220],"TD3","（Twin Delayed DDPG）：Fujimoto 等，2018。",[262,1981,1982,1990],{},[294,1983,1984,1927,1987],{},[339,1985,1986],{},"7",[216,1988,1665],{"href":1663,"rel":1989},[220],"（Soft Actor-Critic）：Haarnoja 等，2018。",[262,1992,1993,2001],{},[294,1994,1995,1927,1998],{},[339,1996,1997],{},"8",[216,1999,1483],{"href":1481,"rel":2000},[220],"（Deep Q-Networks）：Mnih 等，2013。",[262,2003,2004,2012],{},[294,2005,2006,1927,2009],{},[339,2007,2008],{},"9",[216,2010,1491],{"href":1489,"rel":2011},[220],"（Categorical 51-Atom DQN）：Bellemare 等，2017。",[262,2014,2015,2025],{},[294,2016,2017,1927,2020],{},[339,2018,2019],{},"10",[216,2021,2024],{"href":2022,"rel":2023},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1710.10044",[220],"QR-DQN","（Quantile Regression DQN）：Dabney 等，2017。",[262,2027,2028,2038],{},[294,2029,2030,1927,2033],{},[339,2031,2032],{},"11",[216,2034,2037],{"href":2035,"rel":2036},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1707.01495",[220],"HER","（Hindsight Experience Replay）：Andrychowicz 等，2017。",[262,2040,2041,2049],{},[294,2042,2043,1927,2046],{},[339,2044,2045],{},"12",[216,2047,1853],{"href":1851,"rel":2048},[220],"：Ha 和 Schmidhuber，2018。",[262,2051,2052,2060],{},[294,2053,2054,1927,2057],{},[339,2055,2056],{},"13",[216,2058,1872],{"href":1870,"rel":2059},[220],"（Imagination-Augmented Agents）：Weber 等，2017。",[262,2062,2063,2071],{},[294,2064,2065,1927,2068],{},[339,2066,2067],{},"14",[216,2069,1697],{"href":1695,"rel":2070},[220],"（Model-Based RL with Model-Free Fine-Tuning）：Nagabandi 等，2017。",[262,2073,2074,2082],{},[294,2075,2076,1927,2079],{},[339,2077,2078],{},"15",[216,2080,1845],{"href":1843,"rel":2081},[220],"（Model-Based Value Expansion）：Feinberg 等，2018。",[262,2084,2085,2093],{},[294,2086,2087,1927,2090],{},[339,2088,2089],{},"16",[216,2091,285],{"href":283,"rel":2092},[220],"：Silver 等，2017。",{"title":40,"searchDepth":2095,"depth":2096,"links":2097},2,3,[2098,2113],{"id":241,"depth":2095,"text":241,"children":2099},[2100,2101,2107],{"id":274,"depth":2096,"text":274},{"id":304,"depth":2096,"text":304,"children":2102},[2103,2105,2106],{"id":328,"depth":2104,"text":328},4,{"id":1554,"depth":2104,"text":1555},{"id":1638,"depth":2104,"text":1639},{"id":1669,"depth":2096,"text":1669,"children":2108},[2109,2110,2111,2112],{"id":1675,"depth":2104,"text":1676},{"id":1701,"depth":2104,"text":1701},{"id":1824,"depth":2104,"text":1824},{"id":1857,"depth":2104,"text":1857},{"id":1876,"depth":2095,"text":1876},"md",{},{"title":2117,"description":18},[6],"posts\u002Fspinning-up-rl-intro-2",[11,12,13,14],"TQuQJbPBXM7THfsNwYvHRrzgziNXS4WfJi-EvkC9FPE",1790607246373]