[{"data":1,"prerenderedAt":21018},["ShallowReactive",2],{"blog-posts":3,"content-\u002Fposts\u002Fspinning-up-policy-optimization":202},[4,16,21,26,37,43,47,59,66,72,76,81,89,96,104,109,115,121,126,131,138,145,152,158,165,173,179,185,191,197],{"path":5,"title":6,"description":7,"date":8,"categories":9,"tags":10,"draft":15},"\u002Fposts\u002Fspinning-up-policy-optimization","译","OpenAI Spinning Up 教程第三部分的全文翻译：推导最简单的策略梯度、log-derivative trick、EGLP 引理、reward-to-go 与基线，配 PyTorch 示例代码。","2026-09-28 17:49:25","技术",[11,12,13,14],"强化学习","RL","Spinning Up","翻译",false,{"path":17,"title":6,"description":18,"date":19,"categories":9,"tags":20,"draft":15},"\u002Fposts\u002Fspinning-up-rl-intro-2","OpenAI Spinning Up 第二部分的全文翻译：无模型与基于模型的 RL、策略优化与 Q-learning 的取舍，以及规划、专家迭代和经验增强等方法。","2026-09-28 10:20:21",[11,12,13,14],{"path":22,"title":6,"description":23,"date":24,"categories":9,"tags":25,"draft":15},"\u002Fposts\u002Fspinning-up-rl-intro","OpenAI Spinning Up 教程第一部分的全文翻译：智能体与环境、状态与观测、动作空间、策略、轨迹、奖励与回报、价值函数、贝尔曼方程、优势函数，附 MDP 形式化。","2026-09-28 09:29:40",[11,12,13,14],{"path":27,"title":28,"description":29,"date":30,"categories":31,"tags":32,"draft":36},"\u002Fposts\u002Fwhen-machines-come-alive","冰箱之后，生活还会怎样改变？","从厨房里的果蝇想到冰箱、移动互联网与 AI 的多轮调用：AGI 已经到来，只是昂贵的智能会怎样改变生活？","2026-09-25 06:36:22","生活杂谈",[33,34,35],"AI","随笔","科技",true,{"path":38,"title":39,"description":40,"date":41,"categories":40,"tags":42,"draft":36},"\u002Fposts\u002Fmathematical-foundations-of-llms-1","LLM的数学基础（1）：","","2026-09-19 22:59:55",null,{"path":44,"title":45,"description":40,"date":46,"categories":40,"tags":42,"draft":36},"\u002Fposts\u002Ftowards-maximize-serving-capatity","如何在 offline bench 里最大化 LLM serving system 的吞吐","2026-09-19 05:45:05",{"path":48,"title":49,"description":50,"date":51,"categories":52,"tags":53,"draft":15},"\u002Fposts\u002Fllm-memory-compute-tradeoff","LLM 的 Memory-Compute Trade-off：从 MHA 到 MLA、Sparse 与 Linear Attention","ds4.1f 上线了。它的架构 novel 在哪，便宜和快又是从哪省出来的。","2026-09-11 12:00:00","技术杂谈",[54,55,56,57,58],"LLMs","Attention","MLA","DeepSeek","长上下文",{"path":60,"title":61,"description":62,"date":63,"categories":52,"tags":64,"draft":36},"\u002Fposts\u002Fmarkdown-extensions","Markdown 扩展语法速查","本站 Markdown 支持的扩展语法演示与速查：admonition 提示块、GitHub 仓库卡片、spoiler 遮罩、ABC 乐谱、音频播放器、浏览器可运行代码块。","2026-09-01 15:30:00",[65],"博客",{"path":67,"title":68,"description":69,"date":70,"categories":31,"tags":71,"draft":15},"\u002Fposts\u002Fwhy-we-still-need-to-write-a-blog","为什么还需要手写博客？","AI这么强，还要古法写博客吗？","2026-09-01 13:17:53",[65],{"path":73,"title":74,"description":40,"date":75,"categories":40,"tags":42,"draft":36},"\u002Fposts\u002Fllm-serving-1","LLM Serving（一）:以在H200上部署GLM-5.2为例","2026-07-21 06:15:44",{"path":77,"title":78,"description":40,"date":79,"categories":80,"tags":42,"draft":15},"\u002Fposts\u002Finteresting-music-rhythm","有趣的音乐节奏","2026-07-16 12:47:24","音乐",{"path":82,"title":83,"description":84,"date":85,"categories":9,"tags":86,"draft":15},"\u002Fposts\u002Ftowards-monosemanticity","LLM 可解释性（一）：SAE与单语义","读 Anthropic《Towards Monosemanticity》。从难以解释的神经元出发, 讲清superposition为什么让单个神经元混成一团, SAE怎么把多语义的激活分解成单语义特征。","2026-07-12 22:00:00",[54,87,88],"可解释性","SAE",{"path":90,"title":91,"description":92,"date":93,"categories":31,"tags":94,"draft":36},"\u002Fposts\u002Fpiketty-ai-china-k-shaped","用皮凯蒂的镜头看 AI 与中国 K 型社会","一次凌晨的对话整理。从 r>g 出发，看 AI 时代的资本分配、中国的世袭化趋势、体制内的二元劳动力市场，以及一个普通 00 后在一线打拼的真实牌面。","2026-06-24 03:00:00",[34,95,33],"经济",{"path":97,"title":98,"description":40,"date":99,"categories":40,"tags":100,"draft":36},"\u002Fposts\u002Fclaude-code-compact-behavior","cluade code compact 行为","2026-06-23 20:17:42",[101,102,103],"Agent","AI Agent","Claude Code",{"path":105,"title":106,"description":40,"date":107,"categories":9,"tags":108,"draft":15},"\u002Fposts\u002Fclaude-code-settings","Claude Code 配置分享","2026-06-23 14:22:12",[54,102,101,103],{"path":110,"title":111,"description":112,"date":113,"categories":52,"tags":114,"draft":15},"\u002Fposts\u002Fmy-thoughts-on-the-ai-industry","简单聊聊对 AI Agent 行业的理解","写这篇文章的初衷，是我发觉逐渐忘了前两个月的 AI\u002FLLMs 是什么样的能力和情况了。意识中似乎某些在突然加速，但理性和直觉告诉我这不对，作为行内人，我知道这是一个渐进的、累积的过程。我想自己回忆一下，Agent的发展。","2026-06-20 13:14:25",[54,102,101],{"path":116,"title":117,"description":40,"date":118,"categories":9,"tags":119,"draft":15},"\u002Fposts\u002Frotational-positional-encoding","RoPE","2026-06-17 18:31:40",[54,120,117],"Transformer",{"path":122,"title":123,"description":40,"date":124,"categories":9,"tags":125,"draft":15},"\u002Fposts\u002Fintroduction-to-large-language-model","大语言模型入门基础","2026-06-16 22:36:11",[120,54],{"path":127,"title":128,"description":40,"date":129,"categories":52,"tags":130,"draft":15},"\u002Fposts\u002Fthought-about-coding-agent","关于 Coding Agent 使用的遐思","2026-06-03 17:54:20",[101],{"path":132,"title":133,"description":134,"date":135,"categories":9,"tags":136,"draft":15},"\u002Fposts\u002Fabout-ssh-usage","关于SSH的使用——客户端篇","介绍SSH客户端的用法。","2025-06-17 02:44:57",[137],"工具",{"path":139,"title":140,"description":141,"date":142,"categories":52,"tags":143,"draft":15},"\u002Fposts\u002Fopensourcesoftwarestarttutorial","如何开始一个开源项目？","浅谈我对开源项目的认知。","2025-04-17 18:00:00",[144],"开源",{"path":146,"title":147,"description":148,"date":149,"categories":150,"tags":151,"draft":15},"\u002Fposts\u002Fmotivation","渴求动力","有意识的生活并主动地行动会使得生活更美好。","2025-03-05 00:44:57","生活-情感",[34],{"path":153,"title":154,"description":155,"date":156,"categories":150,"tags":157,"draft":15},"\u002Fposts\u002F2024summary","2024年终总结","啊，痛彻的时光；啊，悲怆的人！","2025-01-01 01:00:00",[34],{"path":159,"title":160,"description":161,"date":162,"categories":9,"tags":163,"draft":15},"\u002Fposts\u002Fvps-crash-startup","快速搭建个人VPS","如何快速搭建个人VPS。","2024-12-31 19:00:00",[164],"VPS",{"path":166,"title":167,"description":168,"date":169,"categories":9,"tags":170,"draft":15},"\u002Fposts\u002Fdata-label-software","AI数据（自动）标注软件概述","对市面上目前的AI数据（自动）标注软件进行调研。","2024-12-20 14:00:00",[171,172],"数据标注","深度学习",{"path":174,"title":175,"description":176,"date":177,"categories":150,"tags":178,"draft":15},"\u002Fposts\u002Fself-destruction","生活的崩坏与自我毁灭","是什么在牵引着我？我的行为，我的思想？我的感受？我所处的地方？它们在何处？","2024-12-19 12:00:00",[34],{"path":180,"title":181,"description":182,"date":183,"categories":9,"tags":184,"draft":15},"\u002Fposts\u002Fcs194-196_llmagents","LLM Agents 课程笔记","UCB CS294\u002F194-196 Large Language Model Agents 的课程笔记。","2024-11-30 20:00:00",[101],{"path":186,"title":187,"description":188,"date":189,"categories":150,"tags":190,"draft":15},"\u002Fposts\u002Fwhatsyourworth","价值、位置","你们的生存究竟有何价值？如果毫无价值，你们究竟为何存在？","2024-11-29 02:00:00",[34],{"path":192,"title":193,"description":194,"date":195,"categories":150,"tags":196,"draft":15},"\u002Fposts\u002Fblogrelifever2","重启博客v2","忙里偷闲学习前端，还是决定结合想法手搓一个自己的博客。","2024-11-25 00:00:00",[65],{"path":198,"title":199,"description":200,"date":201,"categories":150,"tags":42,"draft":15},"\u002Fposts\u002Fblogrelife","重启博客","在我的表达欲与现实所需(问题解决记录)驱动下，我决定重写博客。","2024-10-21 20:44:57",{"id":203,"title":6,"body":204,"categories":9,"date":8,"description":7,"draft":15,"extension":21011,"meta":21012,"navigation":36,"path":5,"seo":21013,"stem":21015,"tags":21016,"__hash__":21017},"posts\u002Fposts\u002Fspinning-up-policy-optimization.md",{"type":205,"value":206,"toc":21001},"minimark",[207,243,251,279,288,292,745,748,1164,1312,1315,1318,1321,1654,2269,2392,2639,2645,3347,3760,3766,5212,5215,7531,7803,8389,8491,8657,8660,8674,8686,8691,8764,8796,8813,9261,9266,9291,9294,9453,9502,9507,9866,9874,9879,9883,9889,9999,10315,11896,11902,11905,11908,12529,12580,12595,12598,13613,13620,13623,14154,14161,14169,14177,14181,14191,14198,14233,14236,14251,14254,14269,14277,14280,14313,14821,14824,15924,15959,16190,16390,17595,17598,17601,18249,18322,18432,18435,18928,18930,19501,19574,19580,19865,19873,20334,20616,20898,20903,20987,20990,20997],[208,209,211],"note",{"title":210},"译者说明",[212,213,214,215,222,223,228,229,234,235,238,239,242],"p",{},"本文是 OpenAI ",[216,217,221],"a",{"href":218,"rel":219},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002F",[220],"nofollow","Spinning Up in Deep RL"," 教程第三部分 ",[216,224,227],{"href":225,"rel":226},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Frl_intro3.html",[220],"Part 3: Intro to Policy Optimization"," 的全文翻译，原仓库采用 ",[216,230,233],{"href":231,"rel":232},"https:\u002F\u002Fgithub.com\u002Fopenai\u002Fspinningup\u002Fblob\u002Fmaster\u002FLICENSE",[220],"MIT 协议","。术语约定与",[216,236,237],{"href":22},"第一篇","、",[216,240,241],{"href":17},"第二篇","一致；文中提到的行号指 GitHub 上对应源文件的行号；选读证明与旧版 Tensorflow 内容链接到官方站点。",[212,244,245,246,250],{},"本节讨论策略优化算法的数学基础，并把内容对接到示例代码上。我们会讲",[247,248,249],"strong",{},"策略梯度","（policy gradient）理论里的三个关键结果：",[252,253,254,263,271],"ul",{},[255,256,257,258,262],"li",{},"描述「策略性能对策略参数的梯度」的",[216,259,261],{"href":260},"#%E6%8E%A8%E5%AF%BC%E6%9C%80%E7%AE%80%E5%8D%95%E7%9A%84%E7%AD%96%E7%95%A5%E6%A2%AF%E5%BA%A6","最简单的方程","；",[255,264,265,266,270],{},"一条允许我们从该表达式里",[216,267,269],{"href":268},"#%E5%88%AB%E8%AE%A9%E8%BF%87%E5%8E%BB%E5%B9%B2%E6%89%B0%E4%BD%A0","去掉无用项","的规则；",[255,272,273,274,278],{},"一条允许我们",[216,275,277],{"href":276},"#%E7%AD%96%E7%95%A5%E6%A2%AF%E5%BA%A6%E4%B8%AD%E7%9A%84%E5%9F%BA%E7%BA%BF","加入有用项","的规则。",[212,280,281,282,287],{},"最后把这三个结果拼在一起，给出策略梯度的优势函数形式，也就是我们的 ",[216,283,286],{"href":284,"rel":285},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Falgorithms\u002Fvpg.html",[220],"Vanilla Policy Gradient","（VPG）实现里用的那个版本。",[289,290,291],"h2",{"id":291},"推导最简单的策略梯度",[212,293,294,295,406,407,694,695,739,740,744],{},"这里考虑随机型参数化策略 ",[296,297,300,329],"span",{"className":298},[299],"katex",[296,301,304],{"className":302},[303],"katex-mathml",[305,306,308],"math",{"xmlns":307},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[309,310,311,324],"semantics",{},[312,313,314],"mrow",{},[315,316,317,321],"msub",{},[318,319,320],"mi",{},"π",[318,322,323],{},"θ",[325,326,328],"annotation",{"encoding":327},"application\u002Fx-tex","\\pi_{\\theta}",[296,330,334],{"className":331,"ariaHidden":333},[332],"katex-html","true",[296,335,338,343],{"className":336},[337],"base",[296,339],{"className":340,"style":342},[341],"strut","height:0.5806em;vertical-align:-0.15em;",[296,344,347,352],{"className":345},[346],"mord",[296,348,320],{"className":349,"style":351},[346,350],"mathnormal","margin-right:0.0359em;",[296,353,356],{"className":354},[355],"msupsub",[296,357,361,397],{"className":358},[359,360],"vlist-t","vlist-t2",[296,362,365,392],{"className":363},[364],"vlist-r",[296,366,370],{"className":367,"style":369},[368],"vlist","height:0.3361em;",[296,371,373,378],{"style":372},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[296,374],{"className":375,"style":377},[376],"pstrut","height:2.7em;",[296,379,385],{"className":380},[381,382,383,384],"sizing","reset-size6","size3","mtight",[296,386,388],{"className":387},[346,384],[296,389,323],{"className":390,"style":391},[346,350,384],"margin-right:0.0278em;",[296,393,396],{"className":394},[395],"vlist-s","​",[296,398,400],{"className":399},[364],[296,401,404],{"className":402,"style":403},[368],"height:0.15em;",[296,405],{},"，目标是最大化期望回报 ",[296,408,410,482],{"className":409},[299],[296,411,413],{"className":412},[303],[305,414,415],{"xmlns":307},[309,416,417,479],{},[312,418,419,422,427,433,436,439,464,467,470,472,474,476],{},[318,420,421],{},"J",[423,424,426],"mo",{"stretchy":425},"false","(",[315,428,429,431],{},[318,430,320],{},[318,432,323],{},[423,434,435],{"stretchy":425},")",[423,437,438],{},"=",[318,440,441],{},[442,443,444,450],"munder",{},[423,445,446],{},[318,447,449],{"mathvariant":448},"double-struck","E",[312,451,452,455,458],{},[318,453,454],{},"τ",[423,456,457],{},"∼",[315,459,460,462],{},[318,461,320],{},[318,463,323],{},[423,465,466],{"stretchy":425},"[",[318,468,469],{},"R",[423,471,426],{"stretchy":425},[318,473,454],{},[423,475,435],{"stretchy":425},[423,477,478],{"stretchy":425},"]",[325,480,481],{"encoding":327},"J(\\pi_{\\theta}) = \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}[R(\\tau)]",[296,483,485,559],{"className":484,"ariaHidden":333},[332],[296,486,488,492,496,500,543,547,552,556],{"className":487},[337],[296,489],{"className":490,"style":491},[341],"height:1em;vertical-align:-0.25em;",[296,493,421],{"className":494,"style":495},[346,350],"margin-right:0.0962em;",[296,497,426],{"className":498},[499],"mopen",[296,501,503,506],{"className":502},[346],[296,504,320],{"className":505,"style":351},[346,350],[296,507,509],{"className":508},[355],[296,510,512,535],{"className":511},[359,360],[296,513,515,532],{"className":514},[364],[296,516,518],{"className":517,"style":369},[368],[296,519,520,523],{"style":372},[296,521],{"className":522,"style":377},[376],[296,524,526],{"className":525},[381,382,383,384],[296,527,529],{"className":528},[346,384],[296,530,323],{"className":531,"style":391},[346,350,384],[296,533,396],{"className":534},[395],[296,536,538],{"className":537},[364],[296,539,541],{"className":540,"style":403},[368],[296,542],{},[296,544,435],{"className":545},[546],"mclose",[296,548],{"className":549,"style":551},[550],"mspace","margin-right:0.2778em;",[296,553,438],{"className":554},[555],"mrel",[296,557],{"className":558,"style":551},[550],[296,560,562,566,677,680,684,687,690],{"className":561},[337],[296,563],{"className":564,"style":565},[341],"height:1.5559em;vertical-align:-0.8059em;",[296,567,569],{"className":568},[346],[296,570,574],{"className":571},[572,573],"mop","op-limits",[296,575,577,668],{"className":576},[359,360],[296,578,580,665],{"className":579},[364],[296,581,584,653],{"className":582,"style":583},[368],"height:0.6889em;",[296,585,587,591],{"style":586},"top:-2.4em;margin-left:0em;",[296,588],{"className":589,"style":590},[376],"height:3em;",[296,592,594],{"className":593},[381,382,383,384],[296,595,597,601,604],{"className":596},[346,384],[296,598,454],{"className":599,"style":600},[346,350,384],"margin-right:0.1132em;",[296,602,457],{"className":603},[555,384],[296,605,607,610],{"className":606},[346,384],[296,608,320],{"className":609,"style":351},[346,350,384],[296,611,613],{"className":612},[355],[296,614,616,644],{"className":615},[359,360],[296,617,619,641],{"className":618},[364],[296,620,623],{"className":621,"style":622},[368],"height:0.3448em;",[296,624,626,630],{"style":625},"top:-2.3488em;margin-left:-0.0359em;margin-right:0.0714em;",[296,627],{"className":628,"style":629},[376],"height:2.5em;",[296,631,635],{"className":632},[381,633,634,384],"reset-size3","size1",[296,636,638],{"className":637},[346,384],[296,639,323],{"className":640,"style":391},[346,350,384],[296,642,396],{"className":643},[395],[296,645,647],{"className":646},[364],[296,648,651],{"className":649,"style":650},[368],"height:0.1512em;",[296,652],{},[296,654,656,659],{"style":655},"top:-3em;",[296,657],{"className":658,"style":590},[376],[296,660,661],{},[296,662,449],{"className":663},[572,664],"mathbb",[296,666,396],{"className":667},[395],[296,669,671],{"className":670},[364],[296,672,675],{"className":673,"style":674},[368],"height:0.8059em;",[296,676],{},[296,678,466],{"className":679},[499],[296,681,469],{"className":682,"style":683},[346,350],"margin-right:0.0077em;",[296,685,426],{"className":686},[499],[296,688,454],{"className":689,"style":600},[346,350],[296,691,693],{"className":692},[546],")]","。为推导方便，取 ",[296,696,698,718],{"className":697},[299],[296,699,701],{"className":700},[303],[305,702,703],{"xmlns":307},[309,704,705,715],{},[312,706,707,709,711,713],{},[318,708,469],{},[423,710,426],{"stretchy":425},[318,712,454],{},[423,714,435],{"stretchy":425},[325,716,717],{"encoding":327},"R(\\tau)",[296,719,721],{"className":720,"ariaHidden":333},[332],[296,722,724,727,730,733,736],{"className":723},[337],[296,725],{"className":726,"style":491},[341],[296,728,469],{"className":729,"style":683},[346,350],[296,731,426],{"className":732},[499],[296,734,454],{"className":735,"style":600},[346,350],[296,737,435],{"className":738},[546]," 为",[216,741,743],{"href":742},"\u002Fposts\u002Fspinning-up-rl-intro#%E5%A5%96%E5%8A%B1%E4%B8%8E%E5%9B%9E%E6%8A%A5","有限视界不折扣回报","；换成无限视界折扣回报的情形，推导几乎完全一样。",[212,746,747],{},"我们想用梯度上升来优化策略，比如：",[296,749,752],{"className":750},[751],"katex-display",[296,753,755,830],{"className":754},[299],[296,756,758],{"className":757},[303],[305,759,761],{"xmlns":307,"display":760},"block",[309,762,763,827],{},[312,764,765,781,783,789,791,794],{},[315,766,767,769],{},[318,768,323],{},[312,770,771,774,777],{},[318,772,773],{},"k",[423,775,776],{},"+",[778,779,780],"mn",{},"1",[423,782,438],{},[315,784,785,787],{},[318,786,323],{},[318,788,773],{},[423,790,776],{},[318,792,793],{},"α",[315,795,796,821],{},[312,797,798,806,808,810,816,818],{},[315,799,800,804],{},[318,801,803],{"mathvariant":802},"normal","∇",[318,805,323],{},[318,807,421],{},[423,809,426],{"stretchy":425},[315,811,812,814],{},[318,813,320],{},[318,815,323],{},[423,817,435],{"stretchy":425},[423,819,820],{"fence":333},"∣",[315,822,823,825],{},[318,824,323],{},[318,826,773],{},[325,828,829],{"encoding":327},"\\theta_{k+1} = \\theta_k + \\alpha \\left. \\nabla_{\\theta} J(\\pi_{\\theta}) \\right|_{\\theta_k}",[296,831,833,902,959],{"className":832,"ariaHidden":333},[332],[296,834,836,840,893,896,899],{"className":835},[337],[296,837],{"className":838,"style":839},[341],"height:0.9028em;vertical-align:-0.2083em;",[296,841,843,846],{"className":842},[346],[296,844,323],{"className":845,"style":391},[346,350],[296,847,849],{"className":848},[355],[296,850,852,884],{"className":851},[359,360],[296,853,855,881],{"className":854},[364],[296,856,858],{"className":857,"style":369},[368],[296,859,861,864],{"style":860},"top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;",[296,862],{"className":863,"style":377},[376],[296,865,867],{"className":866},[381,382,383,384],[296,868,870,874,878],{"className":869},[346,384],[296,871,773],{"className":872,"style":873},[346,350,384],"margin-right:0.0315em;",[296,875,776],{"className":876},[877,384],"mbin",[296,879,780],{"className":880},[346,384],[296,882,396],{"className":883},[395],[296,885,887],{"className":886},[364],[296,888,891],{"className":889,"style":890},[368],"height:0.2083em;",[296,892],{},[296,894],{"className":895,"style":551},[550],[296,897,438],{"className":898},[555],[296,900],{"className":901,"style":551},[550],[296,903,905,909,949,953,956],{"className":904},[337],[296,906],{"className":907,"style":908},[341],"height:0.8444em;vertical-align:-0.15em;",[296,910,912,915],{"className":911},[346],[296,913,323],{"className":914,"style":391},[346,350],[296,916,918],{"className":917},[355],[296,919,921,941],{"className":920},[359,360],[296,922,924,938],{"className":923},[364],[296,925,927],{"className":926,"style":369},[368],[296,928,929,932],{"style":860},[296,930],{"className":931,"style":377},[376],[296,933,935],{"className":934},[381,382,383,384],[296,936,773],{"className":937,"style":873},[346,350,384],[296,939,396],{"className":940},[395],[296,942,944],{"className":943},[364],[296,945,947],{"className":946,"style":403},[368],[296,948],{},[296,950],{"className":951,"style":952},[550],"margin-right:0.2222em;",[296,954,776],{"className":955},[877],[296,957],{"className":958,"style":952},[550],[296,960,962,966,970,974],{"className":961},[337],[296,963],{"className":964,"style":965},[341],"height:1.1556em;vertical-align:-0.4056em;",[296,967,793],{"className":968,"style":969},[346,350],"margin-right:0.0037em;",[296,971],{"className":972,"style":973},[550],"margin-right:0.1667em;",[296,975,978,1086],{"className":976},[977],"minner",[296,979,981,985,1029,1032,1035,1078,1081],{"className":980},[977],[296,982],{"className":983},[499,984],"nulldelimiter",[296,986,988,991],{"className":987},[346],[296,989,803],{"className":990},[346],[296,992,994],{"className":993},[355],[296,995,997,1021],{"className":996},[359,360],[296,998,1000,1018],{"className":999},[364],[296,1001,1003],{"className":1002,"style":369},[368],[296,1004,1006,1009],{"style":1005},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[296,1007],{"className":1008,"style":377},[376],[296,1010,1012],{"className":1011},[381,382,383,384],[296,1013,1015],{"className":1014},[346,384],[296,1016,323],{"className":1017,"style":391},[346,350,384],[296,1019,396],{"className":1020},[395],[296,1022,1024],{"className":1023},[364],[296,1025,1027],{"className":1026,"style":403},[368],[296,1028],{},[296,1030,421],{"className":1031,"style":495},[346,350],[296,1033,426],{"className":1034},[499],[296,1036,1038,1041],{"className":1037},[346],[296,1039,320],{"className":1040,"style":351},[346,350],[296,1042,1044],{"className":1043},[355],[296,1045,1047,1070],{"className":1046},[359,360],[296,1048,1050,1067],{"className":1049},[364],[296,1051,1053],{"className":1052,"style":369},[368],[296,1054,1055,1058],{"style":372},[296,1056],{"className":1057,"style":377},[376],[296,1059,1061],{"className":1060},[381,382,383,384],[296,1062,1064],{"className":1063},[346,384],[296,1065,323],{"className":1066,"style":391},[346,350,384],[296,1068,396],{"className":1069},[395],[296,1071,1073],{"className":1072},[364],[296,1074,1076],{"className":1075,"style":403},[368],[296,1077],{},[296,1079,435],{"className":1080},[546],[296,1082,820],{"className":1083,"style":1085},[546,1084],"delimcenter","top:0em;",[296,1087,1089],{"className":1088},[355],[296,1090,1092,1155],{"className":1091},[359,360],[296,1093,1095,1152],{"className":1094},[364],[296,1096,1099],{"className":1097,"style":1098},[368],"height:0.1864em;",[296,1100,1102,1105],{"style":1101},"top:-2.4003em;margin-right:0.05em;",[296,1103],{"className":1104,"style":377},[376],[296,1106,1108],{"className":1107},[381,382,383,384],[296,1109,1111],{"className":1110},[346,384],[296,1112,1114,1117],{"className":1113},[346,384],[296,1115,323],{"className":1116,"style":391},[346,350,384],[296,1118,1120],{"className":1119},[355],[296,1121,1123,1144],{"className":1122},[359,360],[296,1124,1126,1141],{"className":1125},[364],[296,1127,1129],{"className":1128,"style":622},[368],[296,1130,1132,1135],{"style":1131},"top:-2.3488em;margin-left:-0.0278em;margin-right:0.0714em;",[296,1133],{"className":1134,"style":629},[376],[296,1136,1138],{"className":1137},[381,633,634,384],[296,1139,773],{"className":1140,"style":873},[346,350,384],[296,1142,396],{"className":1143},[395],[296,1145,1147],{"className":1146},[364],[296,1148,1150],{"className":1149,"style":650},[368],[296,1151],{},[296,1153,396],{"className":1154},[395],[296,1156,1158],{"className":1157},[364],[296,1159,1162],{"className":1160,"style":1161},[368],"height:0.4056em;",[296,1163],{},[212,1165,1166,1167,1304,1305,1307,1308,1311],{},"策略性能的梯度 ",[296,1168,1170,1200],{"className":1169},[299],[296,1171,1173],{"className":1172},[303],[305,1174,1175],{"xmlns":307},[309,1176,1177,1197],{},[312,1178,1179,1185,1187,1189,1195],{},[315,1180,1181,1183],{},[318,1182,803],{"mathvariant":802},[318,1184,323],{},[318,1186,421],{},[423,1188,426],{"stretchy":425},[315,1190,1191,1193],{},[318,1192,320],{},[318,1194,323],{},[423,1196,435],{"stretchy":425},[325,1198,1199],{"encoding":327},"\\nabla_{\\theta} J(\\pi_{\\theta})",[296,1201,1203],{"className":1202,"ariaHidden":333},[332],[296,1204,1206,1209,1252,1255,1258,1301],{"className":1205},[337],[296,1207],{"className":1208,"style":491},[341],[296,1210,1212,1215],{"className":1211},[346],[296,1213,803],{"className":1214},[346],[296,1216,1218],{"className":1217},[355],[296,1219,1221,1244],{"className":1220},[359,360],[296,1222,1224,1241],{"className":1223},[364],[296,1225,1227],{"className":1226,"style":369},[368],[296,1228,1229,1232],{"style":1005},[296,1230],{"className":1231,"style":377},[376],[296,1233,1235],{"className":1234},[381,382,383,384],[296,1236,1238],{"className":1237},[346,384],[296,1239,323],{"className":1240,"style":391},[346,350,384],[296,1242,396],{"className":1243},[395],[296,1245,1247],{"className":1246},[364],[296,1248,1250],{"className":1249,"style":403},[368],[296,1251],{},[296,1253,421],{"className":1254,"style":495},[346,350],[296,1256,426],{"className":1257},[499],[296,1259,1261,1264],{"className":1260},[346],[296,1262,320],{"className":1263,"style":351},[346,350],[296,1265,1267],{"className":1266},[355],[296,1268,1270,1293],{"className":1269},[359,360],[296,1271,1273,1290],{"className":1272},[364],[296,1274,1276],{"className":1275,"style":369},[368],[296,1277,1278,1281],{"style":372},[296,1279],{"className":1280,"style":377},[376],[296,1282,1284],{"className":1283},[381,382,383,384],[296,1285,1287],{"className":1286},[346,384],[296,1288,323],{"className":1289,"style":391},[346,350,384],[296,1291,396],{"className":1292},[395],[296,1294,1296],{"className":1295},[364],[296,1297,1299],{"className":1298,"style":403},[368],[296,1300],{},[296,1302,435],{"className":1303},[546]," 叫作",[247,1306,249],{},"，按这个方式优化策略的算法叫",[247,1309,1310],{},"策略梯度算法","（例子有 Vanilla Policy Gradient 和 TRPO。PPO 常被叫作策略梯度算法，不过严格说稍有不准）。",[212,1313,1314],{},"要真正用上这个算法，得先有一个能数值计算的策略梯度表达式。这分两步：1）推导策略性能的解析梯度，它最后会化成一个期望的形式；2）给这个期望做样本估计，用有限步「智能体-环境交互」的数据就能算出来。",[212,1316,1317],{},"本小节先找出这个表达式最简单的形式；后面几小节再展示如何在最简单的形式上改进，得到标准策略梯度实现里真正使用的版本。",[212,1319,1320],{},"先摆出几个对推导解析梯度有用的事实。",[212,1322,1323,1326,1327,1399,1400,1653],{},[247,1324,1325],{},"1. 轨迹的概率","。动作来自 ",[296,1328,1330,1347],{"className":1329},[299],[296,1331,1333],{"className":1332},[303],[305,1334,1335],{"xmlns":307},[309,1336,1337,1345],{},[312,1338,1339],{},[315,1340,1341,1343],{},[318,1342,320],{},[318,1344,323],{},[325,1346,328],{"encoding":327},[296,1348,1350],{"className":1349,"ariaHidden":333},[332],[296,1351,1353,1356],{"className":1352},[337],[296,1354],{"className":1355,"style":342},[341],[296,1357,1359,1362],{"className":1358},[346],[296,1360,320],{"className":1361,"style":351},[346,350],[296,1363,1365],{"className":1364},[355],[296,1366,1368,1391],{"className":1367},[359,360],[296,1369,1371,1388],{"className":1370},[364],[296,1372,1374],{"className":1373,"style":369},[368],[296,1375,1376,1379],{"style":372},[296,1377],{"className":1378,"style":377},[376],[296,1380,1382],{"className":1381},[381,382,383,384],[296,1383,1385],{"className":1384},[346,384],[296,1386,323],{"className":1387,"style":391},[346,350,384],[296,1389,396],{"className":1390},[395],[296,1392,1394],{"className":1393},[364],[296,1395,1397],{"className":1396,"style":403},[368],[296,1398],{}," 时，轨迹 ",[296,1401,1403,1464],{"className":1402},[299],[296,1404,1406],{"className":1405},[303],[305,1407,1408],{"xmlns":307},[309,1409,1410,1461],{},[312,1411,1412,1414,1416,1418,1426,1429,1435,1437,1440,1442,1444,1446,1459],{},[318,1413,454],{},[423,1415,438],{},[423,1417,426],{"stretchy":425},[315,1419,1420,1423],{},[318,1421,1422],{},"s",[778,1424,1425],{},"0",[423,1427,1428],{"separator":333},",",[315,1430,1431,1433],{},[318,1432,216],{},[778,1434,1425],{},[423,1436,1428],{"separator":333},[318,1438,1439],{"mathvariant":802},".",[318,1441,1439],{"mathvariant":802},[318,1443,1439],{"mathvariant":802},[423,1445,1428],{"separator":333},[315,1447,1448,1450],{},[318,1449,1422],{},[312,1451,1452,1455,1457],{},[318,1453,1454],{},"T",[423,1456,776],{},[778,1458,780],{},[423,1460,435],{"stretchy":425},[325,1462,1463],{"encoding":327},"\\tau = (s_0, a_0, ..., s_{T+1})",[296,1465,1467,1486],{"className":1466,"ariaHidden":333},[332],[296,1468,1470,1474,1477,1480,1483],{"className":1469},[337],[296,1471],{"className":1472,"style":1473},[341],"height:0.4306em;",[296,1475,454],{"className":1476,"style":600},[346,350],[296,1478],{"className":1479,"style":551},[550],[296,1481,438],{"className":1482},[555],[296,1484],{"className":1485,"style":551},[550],[296,1487,1489,1492,1495,1536,1540,1543,1583,1586,1589,1593,1596,1599,1650],{"className":1488},[337],[296,1490],{"className":1491,"style":491},[341],[296,1493,426],{"className":1494},[499],[296,1496,1498,1501],{"className":1497},[346],[296,1499,1422],{"className":1500},[346,350],[296,1502,1504],{"className":1503},[355],[296,1505,1507,1528],{"className":1506},[359,360],[296,1508,1510,1525],{"className":1509},[364],[296,1511,1514],{"className":1512,"style":1513},[368],"height:0.3011em;",[296,1515,1516,1519],{"style":1005},[296,1517],{"className":1518,"style":377},[376],[296,1520,1522],{"className":1521},[381,382,383,384],[296,1523,1425],{"className":1524},[346,384],[296,1526,396],{"className":1527},[395],[296,1529,1531],{"className":1530},[364],[296,1532,1534],{"className":1533,"style":403},[368],[296,1535],{},[296,1537,1428],{"className":1538},[1539],"mpunct",[296,1541],{"className":1542,"style":973},[550],[296,1544,1546,1549],{"className":1545},[346],[296,1547,216],{"className":1548},[346,350],[296,1550,1552],{"className":1551},[355],[296,1553,1555,1575],{"className":1554},[359,360],[296,1556,1558,1572],{"className":1557},[364],[296,1559,1561],{"className":1560,"style":1513},[368],[296,1562,1563,1566],{"style":1005},[296,1564],{"className":1565,"style":377},[376],[296,1567,1569],{"className":1568},[381,382,383,384],[296,1570,1425],{"className":1571},[346,384],[296,1573,396],{"className":1574},[395],[296,1576,1578],{"className":1577},[364],[296,1579,1581],{"className":1580,"style":403},[368],[296,1582],{},[296,1584,1428],{"className":1585},[1539],[296,1587],{"className":1588,"style":973},[550],[296,1590,1592],{"className":1591},[346],"...",[296,1594,1428],{"className":1595},[1539],[296,1597],{"className":1598,"style":973},[550],[296,1600,1602,1605],{"className":1601},[346],[296,1603,1422],{"className":1604},[346,350],[296,1606,1608],{"className":1607},[355],[296,1609,1611,1642],{"className":1610},[359,360],[296,1612,1614,1639],{"className":1613},[364],[296,1615,1618],{"className":1616,"style":1617},[368],"height:0.3283em;",[296,1619,1620,1623],{"style":1005},[296,1621],{"className":1622,"style":377},[376],[296,1624,1626],{"className":1625},[381,382,383,384],[296,1627,1629,1633,1636],{"className":1628},[346,384],[296,1630,1454],{"className":1631,"style":1632},[346,350,384],"margin-right:0.1389em;",[296,1634,776],{"className":1635},[877,384],[296,1637,780],{"className":1638},[346,384],[296,1640,396],{"className":1641},[395],[296,1643,1645],{"className":1644},[364],[296,1646,1648],{"className":1647,"style":890},[368],[296,1649],{},[296,1651,435],{"className":1652},[546]," 的概率是",[296,1655,1657],{"className":1656},[751],[296,1658,1660,1779],{"className":1659},[299],[296,1661,1663],{"className":1662},[303],[305,1664,1665],{"xmlns":307,"display":760},[309,1666,1667,1776],{},[312,1668,1669,1672,1674,1676,1678,1680,1682,1684,1691,1693,1699,1701,1718,1720,1722,1734,1736,1742,1744,1750,1752,1758,1760,1766,1768,1774],{},[318,1670,1671],{},"P",[423,1673,426],{"stretchy":425},[318,1675,454],{},[318,1677,820],{"mathvariant":802},[318,1679,323],{},[423,1681,435],{"stretchy":425},[423,1683,438],{},[315,1685,1686,1689],{},[318,1687,1688],{},"ρ",[778,1690,1425],{},[423,1692,426],{"stretchy":425},[315,1694,1695,1697],{},[318,1696,1422],{},[778,1698,1425],{},[423,1700,435],{"stretchy":425},[1702,1703,1704,1707,1716],"munderover",{},[423,1705,1706],{},"∏",[312,1708,1709,1712,1714],{},[318,1710,1711],{},"t",[423,1713,438],{},[778,1715,1425],{},[318,1717,1454],{},[318,1719,1671],{},[423,1721,426],{"stretchy":425},[315,1723,1724,1726],{},[318,1725,1422],{},[312,1727,1728,1730,1732],{},[318,1729,1711],{},[423,1731,776],{},[778,1733,780],{},[318,1735,820],{"mathvariant":802},[315,1737,1738,1740],{},[318,1739,1422],{},[318,1741,1711],{},[423,1743,1428],{"separator":333},[315,1745,1746,1748],{},[318,1747,216],{},[318,1749,1711],{},[423,1751,435],{"stretchy":425},[315,1753,1754,1756],{},[318,1755,320],{},[318,1757,323],{},[423,1759,426],{"stretchy":425},[315,1761,1762,1764],{},[318,1763,216],{},[318,1765,1711],{},[318,1767,820],{"mathvariant":802},[315,1769,1770,1772],{},[318,1771,1422],{},[318,1773,1711],{},[423,1775,435],{"stretchy":425},[325,1777,1778],{"encoding":327},"P(\\tau|\\theta) = \\rho_0 (s_0) \\prod_{t=0}^{T} P(s_{t+1}|s_t, a_t) \\pi_{\\theta}(a_t |s_t)",[296,1780,1782,1815],{"className":1781,"ariaHidden":333},[332],[296,1783,1785,1788,1791,1794,1797,1800,1803,1806,1809,1812],{"className":1784},[337],[296,1786],{"className":1787,"style":491},[341],[296,1789,1671],{"className":1790,"style":1632},[346,350],[296,1792,426],{"className":1793},[499],[296,1795,454],{"className":1796,"style":600},[346,350],[296,1798,820],{"className":1799},[346],[296,1801,323],{"className":1802,"style":391},[346,350],[296,1804,435],{"className":1805},[546],[296,1807],{"className":1808,"style":551},[550],[296,1810,438],{"className":1811},[555],[296,1813],{"className":1814,"style":551},[550],[296,1816,1818,1822,1862,1865,1905,1908,1911,1986,1989,1992,1995,2044,2047,2088,2091,2094,2134,2137,2180,2183,2223,2226,2266],{"className":1817},[337],[296,1819],{"className":1820,"style":1821},[341],"height:3.0954em;vertical-align:-1.2671em;",[296,1823,1825,1828],{"className":1824},[346],[296,1826,1688],{"className":1827},[346,350],[296,1829,1831],{"className":1830},[355],[296,1832,1834,1854],{"className":1833},[359,360],[296,1835,1837,1851],{"className":1836},[364],[296,1838,1840],{"className":1839,"style":1513},[368],[296,1841,1842,1845],{"style":1005},[296,1843],{"className":1844,"style":377},[376],[296,1846,1848],{"className":1847},[381,382,383,384],[296,1849,1425],{"className":1850},[346,384],[296,1852,396],{"className":1853},[395],[296,1855,1857],{"className":1856},[364],[296,1858,1860],{"className":1859,"style":403},[368],[296,1861],{},[296,1863,426],{"className":1864},[499],[296,1866,1868,1871],{"className":1867},[346],[296,1869,1422],{"className":1870},[346,350],[296,1872,1874],{"className":1873},[355],[296,1875,1877,1897],{"className":1876},[359,360],[296,1878,1880,1894],{"className":1879},[364],[296,1881,1883],{"className":1882,"style":1513},[368],[296,1884,1885,1888],{"style":1005},[296,1886],{"className":1887,"style":377},[376],[296,1889,1891],{"className":1890},[381,382,383,384],[296,1892,1425],{"className":1893},[346,384],[296,1895,396],{"className":1896},[395],[296,1898,1900],{"className":1899},[364],[296,1901,1903],{"className":1902,"style":403},[368],[296,1904],{},[296,1906,435],{"className":1907},[546],[296,1909],{"className":1910,"style":973},[550],[296,1912,1914],{"className":1913},[572,573],[296,1915,1917,1977],{"className":1916},[359,360],[296,1918,1920,1974],{"className":1919},[364],[296,1921,1924,1946,1959],{"className":1922,"style":1923},[368],"height:1.8283em;",[296,1925,1927,1931],{"style":1926},"top:-1.8829em;margin-left:0em;",[296,1928],{"className":1929,"style":1930},[376],"height:3.05em;",[296,1932,1934],{"className":1933},[381,382,383,384],[296,1935,1937,1940,1943],{"className":1936},[346,384],[296,1938,1711],{"className":1939},[346,350,384],[296,1941,438],{"className":1942},[555,384],[296,1944,1425],{"className":1945},[346,384],[296,1947,1949,1952],{"style":1948},"top:-3.05em;",[296,1950],{"className":1951,"style":1930},[376],[296,1953,1954],{},[296,1955,1706],{"className":1956},[572,1957,1958],"op-symbol","large-op",[296,1960,1962,1965],{"style":1961},"top:-4.3em;margin-left:0em;",[296,1963],{"className":1964,"style":1930},[376],[296,1966,1968],{"className":1967},[381,382,383,384],[296,1969,1971],{"className":1970},[346,384],[296,1972,1454],{"className":1973,"style":1632},[346,350,384],[296,1975,396],{"className":1976},[395],[296,1978,1980],{"className":1979},[364],[296,1981,1984],{"className":1982,"style":1983},[368],"height:1.2671em;",[296,1985],{},[296,1987],{"className":1988,"style":973},[550],[296,1990,1671],{"className":1991,"style":1632},[346,350],[296,1993,426],{"className":1994},[499],[296,1996,1998,2001],{"className":1997},[346],[296,1999,1422],{"className":2000},[346,350],[296,2002,2004],{"className":2003},[355],[296,2005,2007,2036],{"className":2006},[359,360],[296,2008,2010,2033],{"className":2009},[364],[296,2011,2013],{"className":2012,"style":1513},[368],[296,2014,2015,2018],{"style":1005},[296,2016],{"className":2017,"style":377},[376],[296,2019,2021],{"className":2020},[381,382,383,384],[296,2022,2024,2027,2030],{"className":2023},[346,384],[296,2025,1711],{"className":2026},[346,350,384],[296,2028,776],{"className":2029},[877,384],[296,2031,780],{"className":2032},[346,384],[296,2034,396],{"className":2035},[395],[296,2037,2039],{"className":2038},[364],[296,2040,2042],{"className":2041,"style":890},[368],[296,2043],{},[296,2045,820],{"className":2046},[346],[296,2048,2050,2053],{"className":2049},[346],[296,2051,1422],{"className":2052},[346,350],[296,2054,2056],{"className":2055},[355],[296,2057,2059,2080],{"className":2058},[359,360],[296,2060,2062,2077],{"className":2061},[364],[296,2063,2066],{"className":2064,"style":2065},[368],"height:0.2806em;",[296,2067,2068,2071],{"style":1005},[296,2069],{"className":2070,"style":377},[376],[296,2072,2074],{"className":2073},[381,382,383,384],[296,2075,1711],{"className":2076},[346,350,384],[296,2078,396],{"className":2079},[395],[296,2081,2083],{"className":2082},[364],[296,2084,2086],{"className":2085,"style":403},[368],[296,2087],{},[296,2089,1428],{"className":2090},[1539],[296,2092],{"className":2093,"style":973},[550],[296,2095,2097,2100],{"className":2096},[346],[296,2098,216],{"className":2099},[346,350],[296,2101,2103],{"className":2102},[355],[296,2104,2106,2126],{"className":2105},[359,360],[296,2107,2109,2123],{"className":2108},[364],[296,2110,2112],{"className":2111,"style":2065},[368],[296,2113,2114,2117],{"style":1005},[296,2115],{"className":2116,"style":377},[376],[296,2118,2120],{"className":2119},[381,382,383,384],[296,2121,1711],{"className":2122},[346,350,384],[296,2124,396],{"className":2125},[395],[296,2127,2129],{"className":2128},[364],[296,2130,2132],{"className":2131,"style":403},[368],[296,2133],{},[296,2135,435],{"className":2136},[546],[296,2138,2140,2143],{"className":2139},[346],[296,2141,320],{"className":2142,"style":351},[346,350],[296,2144,2146],{"className":2145},[355],[296,2147,2149,2172],{"className":2148},[359,360],[296,2150,2152,2169],{"className":2151},[364],[296,2153,2155],{"className":2154,"style":369},[368],[296,2156,2157,2160],{"style":372},[296,2158],{"className":2159,"style":377},[376],[296,2161,2163],{"className":2162},[381,382,383,384],[296,2164,2166],{"className":2165},[346,384],[296,2167,323],{"className":2168,"style":391},[346,350,384],[296,2170,396],{"className":2171},[395],[296,2173,2175],{"className":2174},[364],[296,2176,2178],{"className":2177,"style":403},[368],[296,2179],{},[296,2181,426],{"className":2182},[499],[296,2184,2186,2189],{"className":2185},[346],[296,2187,216],{"className":2188},[346,350],[296,2190,2192],{"className":2191},[355],[296,2193,2195,2215],{"className":2194},[359,360],[296,2196,2198,2212],{"className":2197},[364],[296,2199,2201],{"className":2200,"style":2065},[368],[296,2202,2203,2206],{"style":1005},[296,2204],{"className":2205,"style":377},[376],[296,2207,2209],{"className":2208},[381,382,383,384],[296,2210,1711],{"className":2211},[346,350,384],[296,2213,396],{"className":2214},[395],[296,2216,2218],{"className":2217},[364],[296,2219,2221],{"className":2220,"style":403},[368],[296,2222],{},[296,2224,820],{"className":2225},[346],[296,2227,2229,2232],{"className":2228},[346],[296,2230,1422],{"className":2231},[346,350],[296,2233,2235],{"className":2234},[355],[296,2236,2238,2258],{"className":2237},[359,360],[296,2239,2241,2255],{"className":2240},[364],[296,2242,2244],{"className":2243,"style":2065},[368],[296,2245,2246,2249],{"style":1005},[296,2247],{"className":2248,"style":377},[376],[296,2250,2252],{"className":2251},[381,382,383,384],[296,2253,1711],{"className":2254},[346,350,384],[296,2256,396],{"className":2257},[395],[296,2259,2261],{"className":2260},[364],[296,2262,2264],{"className":2263,"style":403},[368],[296,2265],{},[296,2267,435],{"className":2268},[546],[212,2270,2271,2274,2275,2323,2324,2352,2353,2391],{},[247,2272,2273],{},"2. 对数求导技巧（log-derivative trick）","。它基于微积分里一条简单的规则：",[296,2276,2278,2299],{"className":2277},[299],[296,2279,2281],{"className":2280},[303],[305,2282,2283],{"xmlns":307},[309,2284,2285,2296],{},[312,2286,2287,2290,2293],{},[318,2288,2289],{},"log",[423,2291,2292],{},"⁡",[318,2294,2295],{},"x",[325,2297,2298],{"encoding":327},"\\log x",[296,2300,2302],{"className":2301,"ariaHidden":333},[332],[296,2303,2305,2309,2317,2320],{"className":2304},[337],[296,2306],{"className":2307,"style":2308},[341],"height:0.8889em;vertical-align:-0.1944em;",[296,2310,2312,2313],{"className":2311},[572],"lo",[296,2314,2316],{"style":2315},"margin-right:0.0139em;","g",[296,2318],{"className":2319,"style":973},[550],[296,2321,2295],{"className":2322},[346,350]," 对 ",[296,2325,2327,2340],{"className":2326},[299],[296,2328,2330],{"className":2329},[303],[305,2331,2332],{"xmlns":307},[309,2333,2334,2338],{},[312,2335,2336],{},[318,2337,2295],{},[325,2339,2295],{"encoding":327},[296,2341,2343],{"className":2342,"ariaHidden":333},[332],[296,2344,2346,2349],{"className":2345},[337],[296,2347],{"className":2348,"style":1473},[341],[296,2350,2295],{"className":2351},[346,350]," 的导数是 ",[296,2354,2356,2375],{"className":2355},[299],[296,2357,2359],{"className":2358},[303],[305,2360,2361],{"xmlns":307},[309,2362,2363,2372],{},[312,2364,2365,2367,2370],{},[778,2366,780],{},[318,2368,2369],{"mathvariant":802},"\u002F",[318,2371,2295],{},[325,2373,2374],{"encoding":327},"1\u002Fx",[296,2376,2378],{"className":2377,"ariaHidden":333},[332],[296,2379,2381,2384,2388],{"className":2380},[337],[296,2382],{"className":2383,"style":491},[341],[296,2385,2387],{"className":2386},[346],"1\u002F",[296,2389,2295],{"className":2390},[346,350],"。稍作变形、再与链式法则结合，得到：",[296,2393,2395],{"className":2394},[751],[296,2396,2398,2464],{"className":2397},[299],[296,2399,2401],{"className":2400},[303],[305,2402,2403],{"xmlns":307,"display":760},[309,2404,2405,2461],{},[312,2406,2407,2413,2415,2417,2419,2421,2423,2425,2427,2429,2431,2433,2435,2437,2439,2445,2447,2449,2451,2453,2455,2457,2459],{},[315,2408,2409,2411],{},[318,2410,803],{"mathvariant":802},[318,2412,323],{},[318,2414,1671],{},[423,2416,426],{"stretchy":425},[318,2418,454],{},[318,2420,820],{"mathvariant":802},[318,2422,323],{},[423,2424,435],{"stretchy":425},[423,2426,438],{},[318,2428,1671],{},[423,2430,426],{"stretchy":425},[318,2432,454],{},[318,2434,820],{"mathvariant":802},[318,2436,323],{},[423,2438,435],{"stretchy":425},[315,2440,2441,2443],{},[318,2442,803],{"mathvariant":802},[318,2444,323],{},[318,2446,2289],{},[423,2448,2292],{},[318,2450,1671],{},[423,2452,426],{"stretchy":425},[318,2454,454],{},[318,2456,820],{"mathvariant":802},[318,2458,323],{},[423,2460,435],{"stretchy":425},[325,2462,2463],{"encoding":327},"\\nabla_{\\theta} P(\\tau | \\theta) = P(\\tau | \\theta) \\nabla_{\\theta} \\log P(\\tau | \\theta)",[296,2465,2467,2543],{"className":2466,"ariaHidden":333},[332],[296,2468,2470,2473,2516,2519,2522,2525,2528,2531,2534,2537,2540],{"className":2469},[337],[296,2471],{"className":2472,"style":491},[341],[296,2474,2476,2479],{"className":2475},[346],[296,2477,803],{"className":2478},[346],[296,2480,2482],{"className":2481},[355],[296,2483,2485,2508],{"className":2484},[359,360],[296,2486,2488,2505],{"className":2487},[364],[296,2489,2491],{"className":2490,"style":369},[368],[296,2492,2493,2496],{"style":1005},[296,2494],{"className":2495,"style":377},[376],[296,2497,2499],{"className":2498},[381,382,383,384],[296,2500,2502],{"className":2501},[346,384],[296,2503,323],{"className":2504,"style":391},[346,350,384],[296,2506,396],{"className":2507},[395],[296,2509,2511],{"className":2510},[364],[296,2512,2514],{"className":2513,"style":403},[368],[296,2515],{},[296,2517,1671],{"className":2518,"style":1632},[346,350],[296,2520,426],{"className":2521},[499],[296,2523,454],{"className":2524,"style":600},[346,350],[296,2526,820],{"className":2527},[346],[296,2529,323],{"className":2530,"style":391},[346,350],[296,2532,435],{"className":2533},[546],[296,2535],{"className":2536,"style":551},[550],[296,2538,438],{"className":2539},[555],[296,2541],{"className":2542,"style":551},[550],[296,2544,2546,2549,2552,2555,2558,2561,2564,2567,2610,2613,2618,2621,2624,2627,2630,2633,2636],{"className":2545},[337],[296,2547],{"className":2548,"style":491},[341],[296,2550,1671],{"className":2551,"style":1632},[346,350],[296,2553,426],{"className":2554},[499],[296,2556,454],{"className":2557,"style":600},[346,350],[296,2559,820],{"className":2560},[346],[296,2562,323],{"className":2563,"style":391},[346,350],[296,2565,435],{"className":2566},[546],[296,2568,2570,2573],{"className":2569},[346],[296,2571,803],{"className":2572},[346],[296,2574,2576],{"className":2575},[355],[296,2577,2579,2602],{"className":2578},[359,360],[296,2580,2582,2599],{"className":2581},[364],[296,2583,2585],{"className":2584,"style":369},[368],[296,2586,2587,2590],{"style":1005},[296,2588],{"className":2589,"style":377},[376],[296,2591,2593],{"className":2592},[381,382,383,384],[296,2594,2596],{"className":2595},[346,384],[296,2597,323],{"className":2598,"style":391},[346,350,384],[296,2600,396],{"className":2601},[395],[296,2603,2605],{"className":2604},[364],[296,2606,2608],{"className":2607,"style":403},[368],[296,2609],{},[296,2611],{"className":2612,"style":973},[550],[296,2614,2312,2616],{"className":2615},[572],[296,2617,2316],{"style":2315},[296,2619],{"className":2620,"style":973},[550],[296,2622,1671],{"className":2623,"style":1632},[346,350],[296,2625,426],{"className":2626},[499],[296,2628,454],{"className":2629,"style":600},[346,350],[296,2631,820],{"className":2632},[346],[296,2634,323],{"className":2635,"style":391},[346,350],[296,2637,435],{"className":2638},[546],[212,2640,2641,2644],{},[247,2642,2643],{},"3. 轨迹的对数概率","。轨迹的 log 概率就是",[296,2646,2648],{"className":2647},[751],[296,2649,2651,2791],{"className":2650},[299],[296,2652,2654],{"className":2653},[303],[305,2655,2656],{"xmlns":307,"display":760},[309,2657,2658,2788],{},[312,2659,2660,2662,2664,2666,2668,2670,2672,2674,2676,2678,2680,2682,2688,2690,2696,2698,2700,2715,2718,2720,2722,2724,2726,2738,2740,2746,2748,2754,2756,2758,2760,2762,2768,2770,2776,2778,2784,2786],{},[318,2661,2289],{},[423,2663,2292],{},[318,2665,1671],{},[423,2667,426],{"stretchy":425},[318,2669,454],{},[318,2671,820],{"mathvariant":802},[318,2673,323],{},[423,2675,435],{"stretchy":425},[423,2677,438],{},[318,2679,2289],{},[423,2681,2292],{},[315,2683,2684,2686],{},[318,2685,1688],{},[778,2687,1425],{},[423,2689,426],{"stretchy":425},[315,2691,2692,2694],{},[318,2693,1422],{},[778,2695,1425],{},[423,2697,435],{"stretchy":425},[423,2699,776],{},[1702,2701,2702,2705,2713],{},[423,2703,2704],{},"∑",[312,2706,2707,2709,2711],{},[318,2708,1711],{},[423,2710,438],{},[778,2712,1425],{},[318,2714,1454],{},[423,2716,426],{"fence":425,"stretchy":333,"minsize":2717,"maxsize":2717},"2.4em",[318,2719,2289],{},[423,2721,2292],{},[318,2723,1671],{},[423,2725,426],{"stretchy":425},[315,2727,2728,2730],{},[318,2729,1422],{},[312,2731,2732,2734,2736],{},[318,2733,1711],{},[423,2735,776],{},[778,2737,780],{},[318,2739,820],{"mathvariant":802},[315,2741,2742,2744],{},[318,2743,1422],{},[318,2745,1711],{},[423,2747,1428],{"separator":333},[315,2749,2750,2752],{},[318,2751,216],{},[318,2753,1711],{},[423,2755,435],{"stretchy":425},[423,2757,776],{},[318,2759,2289],{},[423,2761,2292],{},[315,2763,2764,2766],{},[318,2765,320],{},[318,2767,323],{},[423,2769,426],{"stretchy":425},[315,2771,2772,2774],{},[318,2773,216],{},[318,2775,1711],{},[318,2777,820],{"mathvariant":802},[315,2779,2780,2782],{},[318,2781,1422],{},[318,2783,1711],{},[423,2785,435],{"stretchy":425},[423,2787,435],{"fence":425,"stretchy":333,"minsize":2717,"maxsize":2717},[325,2789,2790],{"encoding":327},"\\log P(\\tau|\\theta) = \\log \\rho_0 (s_0) + \\sum_{t=0}^{T} \\bigg( \\log P(s_{t+1}|s_t, a_t)  + \\log \\pi_{\\theta}(a_t |s_t)\\bigg)",[296,2792,2794,2835,2944,3194],{"className":2793,"ariaHidden":333},[332],[296,2795,2797,2800,2805,2808,2811,2814,2817,2820,2823,2826,2829,2832],{"className":2796},[337],[296,2798],{"className":2799,"style":491},[341],[296,2801,2312,2803],{"className":2802},[572],[296,2804,2316],{"style":2315},[296,2806],{"className":2807,"style":973},[550],[296,2809,1671],{"className":2810,"style":1632},[346,350],[296,2812,426],{"className":2813},[499],[296,2815,454],{"className":2816,"style":600},[346,350],[296,2818,820],{"className":2819},[346],[296,2821,323],{"className":2822,"style":391},[346,350],[296,2824,435],{"className":2825},[546],[296,2827],{"className":2828,"style":551},[550],[296,2830,438],{"className":2831},[555],[296,2833],{"className":2834,"style":551},[550],[296,2836,2838,2841,2846,2849,2889,2892,2932,2935,2938,2941],{"className":2837},[337],[296,2839],{"className":2840,"style":491},[341],[296,2842,2312,2844],{"className":2843},[572],[296,2845,2316],{"style":2315},[296,2847],{"className":2848,"style":973},[550],[296,2850,2852,2855],{"className":2851},[346],[296,2853,1688],{"className":2854},[346,350],[296,2856,2858],{"className":2857},[355],[296,2859,2861,2881],{"className":2860},[359,360],[296,2862,2864,2878],{"className":2863},[364],[296,2865,2867],{"className":2866,"style":1513},[368],[296,2868,2869,2872],{"style":1005},[296,2870],{"className":2871,"style":377},[376],[296,2873,2875],{"className":2874},[381,382,383,384],[296,2876,1425],{"className":2877},[346,384],[296,2879,396],{"className":2880},[395],[296,2882,2884],{"className":2883},[364],[296,2885,2887],{"className":2886,"style":403},[368],[296,2888],{},[296,2890,426],{"className":2891},[499],[296,2893,2895,2898],{"className":2894},[346],[296,2896,1422],{"className":2897},[346,350],[296,2899,2901],{"className":2900},[355],[296,2902,2904,2924],{"className":2903},[359,360],[296,2905,2907,2921],{"className":2906},[364],[296,2908,2910],{"className":2909,"style":1513},[368],[296,2911,2912,2915],{"style":1005},[296,2913],{"className":2914,"style":377},[376],[296,2916,2918],{"className":2917},[381,382,383,384],[296,2919,1425],{"className":2920},[346,384],[296,2922,396],{"className":2923},[395],[296,2925,2927],{"className":2926},[364],[296,2928,2930],{"className":2929,"style":403},[368],[296,2931],{},[296,2933,435],{"className":2934},[546],[296,2936],{"className":2937,"style":952},[550],[296,2939,776],{"className":2940},[877],[296,2942],{"className":2943,"style":952},[550],[296,2945,2947,2950,3017,3020,3027,3030,3035,3038,3041,3044,3093,3096,3136,3139,3142,3182,3185,3188,3191],{"className":2946},[337],[296,2948],{"className":2949,"style":1821},[341],[296,2951,2953],{"className":2952},[572,573],[296,2954,2956,3009],{"className":2955},[359,360],[296,2957,2959,3006],{"className":2958},[364],[296,2960,2962,2982,2992],{"className":2961,"style":1923},[368],[296,2963,2964,2967],{"style":1926},[296,2965],{"className":2966,"style":1930},[376],[296,2968,2970],{"className":2969},[381,382,383,384],[296,2971,2973,2976,2979],{"className":2972},[346,384],[296,2974,1711],{"className":2975},[346,350,384],[296,2977,438],{"className":2978},[555,384],[296,2980,1425],{"className":2981},[346,384],[296,2983,2984,2987],{"style":1948},[296,2985],{"className":2986,"style":1930},[376],[296,2988,2989],{},[296,2990,2704],{"className":2991},[572,1957,1958],[296,2993,2994,2997],{"style":1961},[296,2995],{"className":2996,"style":1930},[376],[296,2998,3000],{"className":2999},[381,382,383,384],[296,3001,3003],{"className":3002},[346,384],[296,3004,1454],{"className":3005,"style":1632},[346,350,384],[296,3007,396],{"className":3008},[395],[296,3010,3012],{"className":3011},[364],[296,3013,3015],{"className":3014,"style":1983},[368],[296,3016],{},[296,3018],{"className":3019,"style":973},[550],[296,3021,3023],{"className":3022},[346],[296,3024,426],{"className":3025},[3026,383],"delimsizing",[296,3028],{"className":3029,"style":973},[550],[296,3031,2312,3033],{"className":3032},[572],[296,3034,2316],{"style":2315},[296,3036],{"className":3037,"style":973},[550],[296,3039,1671],{"className":3040,"style":1632},[346,350],[296,3042,426],{"className":3043},[499],[296,3045,3047,3050],{"className":3046},[346],[296,3048,1422],{"className":3049},[346,350],[296,3051,3053],{"className":3052},[355],[296,3054,3056,3085],{"className":3055},[359,360],[296,3057,3059,3082],{"className":3058},[364],[296,3060,3062],{"className":3061,"style":1513},[368],[296,3063,3064,3067],{"style":1005},[296,3065],{"className":3066,"style":377},[376],[296,3068,3070],{"className":3069},[381,382,383,384],[296,3071,3073,3076,3079],{"className":3072},[346,384],[296,3074,1711],{"className":3075},[346,350,384],[296,3077,776],{"className":3078},[877,384],[296,3080,780],{"className":3081},[346,384],[296,3083,396],{"className":3084},[395],[296,3086,3088],{"className":3087},[364],[296,3089,3091],{"className":3090,"style":890},[368],[296,3092],{},[296,3094,820],{"className":3095},[346],[296,3097,3099,3102],{"className":3098},[346],[296,3100,1422],{"className":3101},[346,350],[296,3103,3105],{"className":3104},[355],[296,3106,3108,3128],{"className":3107},[359,360],[296,3109,3111,3125],{"className":3110},[364],[296,3112,3114],{"className":3113,"style":2065},[368],[296,3115,3116,3119],{"style":1005},[296,3117],{"className":3118,"style":377},[376],[296,3120,3122],{"className":3121},[381,382,383,384],[296,3123,1711],{"className":3124},[346,350,384],[296,3126,396],{"className":3127},[395],[296,3129,3131],{"className":3130},[364],[296,3132,3134],{"className":3133,"style":403},[368],[296,3135],{},[296,3137,1428],{"className":3138},[1539],[296,3140],{"className":3141,"style":973},[550],[296,3143,3145,3148],{"className":3144},[346],[296,3146,216],{"className":3147},[346,350],[296,3149,3151],{"className":3150},[355],[296,3152,3154,3174],{"className":3153},[359,360],[296,3155,3157,3171],{"className":3156},[364],[296,3158,3160],{"className":3159,"style":2065},[368],[296,3161,3162,3165],{"style":1005},[296,3163],{"className":3164,"style":377},[376],[296,3166,3168],{"className":3167},[381,382,383,384],[296,3169,1711],{"className":3170},[346,350,384],[296,3172,396],{"className":3173},[395],[296,3175,3177],{"className":3176},[364],[296,3178,3180],{"className":3179,"style":403},[368],[296,3181],{},[296,3183,435],{"className":3184},[546],[296,3186],{"className":3187,"style":952},[550],[296,3189,776],{"className":3190},[877],[296,3192],{"className":3193,"style":952},[550],[296,3195,3197,3201,3206,3209,3252,3255,3295,3298,3338,3341],{"className":3196},[337],[296,3198],{"className":3199,"style":3200},[341],"height:2.4em;vertical-align:-0.95em;",[296,3202,2312,3204],{"className":3203},[572],[296,3205,2316],{"style":2315},[296,3207],{"className":3208,"style":973},[550],[296,3210,3212,3215],{"className":3211},[346],[296,3213,320],{"className":3214,"style":351},[346,350],[296,3216,3218],{"className":3217},[355],[296,3219,3221,3244],{"className":3220},[359,360],[296,3222,3224,3241],{"className":3223},[364],[296,3225,3227],{"className":3226,"style":369},[368],[296,3228,3229,3232],{"style":372},[296,3230],{"className":3231,"style":377},[376],[296,3233,3235],{"className":3234},[381,382,383,384],[296,3236,3238],{"className":3237},[346,384],[296,3239,323],{"className":3240,"style":391},[346,350,384],[296,3242,396],{"className":3243},[395],[296,3245,3247],{"className":3246},[364],[296,3248,3250],{"className":3249,"style":403},[368],[296,3251],{},[296,3253,426],{"className":3254},[499],[296,3256,3258,3261],{"className":3257},[346],[296,3259,216],{"className":3260},[346,350],[296,3262,3264],{"className":3263},[355],[296,3265,3267,3287],{"className":3266},[359,360],[296,3268,3270,3284],{"className":3269},[364],[296,3271,3273],{"className":3272,"style":2065},[368],[296,3274,3275,3278],{"style":1005},[296,3276],{"className":3277,"style":377},[376],[296,3279,3281],{"className":3280},[381,382,383,384],[296,3282,1711],{"className":3283},[346,350,384],[296,3285,396],{"className":3286},[395],[296,3288,3290],{"className":3289},[364],[296,3291,3293],{"className":3292,"style":403},[368],[296,3294],{},[296,3296,820],{"className":3297},[346],[296,3299,3301,3304],{"className":3300},[346],[296,3302,1422],{"className":3303},[346,350],[296,3305,3307],{"className":3306},[355],[296,3308,3310,3330],{"className":3309},[359,360],[296,3311,3313,3327],{"className":3312},[364],[296,3314,3316],{"className":3315,"style":2065},[368],[296,3317,3318,3321],{"style":1005},[296,3319],{"className":3320,"style":377},[376],[296,3322,3324],{"className":3323},[381,382,383,384],[296,3325,1711],{"className":3326},[346,350,384],[296,3328,396],{"className":3329},[395],[296,3331,3333],{"className":3332},[364],[296,3334,3336],{"className":3335,"style":403},[368],[296,3337],{},[296,3339,435],{"className":3340},[546],[296,3342,3344],{"className":3343},[346],[296,3345,435],{"className":3346},[3026,383],[212,3348,3349,3352,3353,3383,3384,238,3510,3715,3716,3759],{},[247,3350,3351],{},"4. 环境函数的梯度","。环境与 ",[296,3354,3356,3370],{"className":3355},[299],[296,3357,3359],{"className":3358},[303],[305,3360,3361],{"xmlns":307},[309,3362,3363,3367],{},[312,3364,3365],{},[318,3366,323],{},[325,3368,3369],{"encoding":327},"\\theta",[296,3371,3373],{"className":3372,"ariaHidden":333},[332],[296,3374,3376,3380],{"className":3375},[337],[296,3377],{"className":3378,"style":3379},[341],"height:0.6944em;",[296,3381,323],{"className":3382,"style":391},[346,350]," 无关，所以 ",[296,3385,3387,3415],{"className":3386},[299],[296,3388,3390],{"className":3389},[303],[305,3391,3392],{"xmlns":307},[309,3393,3394,3412],{},[312,3395,3396,3402,3404,3410],{},[315,3397,3398,3400],{},[318,3399,1688],{},[778,3401,1425],{},[423,3403,426],{"stretchy":425},[315,3405,3406,3408],{},[318,3407,1422],{},[778,3409,1425],{},[423,3411,435],{"stretchy":425},[325,3413,3414],{"encoding":327},"\\rho_0(s_0)",[296,3416,3418],{"className":3417,"ariaHidden":333},[332],[296,3419,3421,3424,3464,3467,3507],{"className":3420},[337],[296,3422],{"className":3423,"style":491},[341],[296,3425,3427,3430],{"className":3426},[346],[296,3428,1688],{"className":3429},[346,350],[296,3431,3433],{"className":3432},[355],[296,3434,3436,3456],{"className":3435},[359,360],[296,3437,3439,3453],{"className":3438},[364],[296,3440,3442],{"className":3441,"style":1513},[368],[296,3443,3444,3447],{"style":1005},[296,3445],{"className":3446,"style":377},[376],[296,3448,3450],{"className":3449},[381,382,383,384],[296,3451,1425],{"className":3452},[346,384],[296,3454,396],{"className":3455},[395],[296,3457,3459],{"className":3458},[364],[296,3460,3462],{"className":3461,"style":403},[368],[296,3463],{},[296,3465,426],{"className":3466},[499],[296,3468,3470,3473],{"className":3469},[346],[296,3471,1422],{"className":3472},[346,350],[296,3474,3476],{"className":3475},[355],[296,3477,3479,3499],{"className":3478},[359,360],[296,3480,3482,3496],{"className":3481},[364],[296,3483,3485],{"className":3484,"style":1513},[368],[296,3486,3487,3490],{"style":1005},[296,3488],{"className":3489,"style":377},[376],[296,3491,3493],{"className":3492},[381,382,383,384],[296,3494,1425],{"className":3495},[346,384],[296,3497,396],{"className":3498},[395],[296,3500,3502],{"className":3501},[364],[296,3503,3505],{"className":3504,"style":403},[368],[296,3506],{},[296,3508,435],{"className":3509},[546],[296,3511,3513,3559],{"className":3512},[299],[296,3514,3516],{"className":3515},[303],[305,3517,3518],{"xmlns":307},[309,3519,3520,3556],{},[312,3521,3522,3524,3526,3538,3540,3546,3548,3554],{},[318,3523,1671],{},[423,3525,426],{"stretchy":425},[315,3527,3528,3530],{},[318,3529,1422],{},[312,3531,3532,3534,3536],{},[318,3533,1711],{},[423,3535,776],{},[778,3537,780],{},[318,3539,820],{"mathvariant":802},[315,3541,3542,3544],{},[318,3543,1422],{},[318,3545,1711],{},[423,3547,1428],{"separator":333},[315,3549,3550,3552],{},[318,3551,216],{},[318,3553,1711],{},[423,3555,435],{"stretchy":425},[325,3557,3558],{"encoding":327},"P(s_{t+1}|s_t, a_t)",[296,3560,3562],{"className":3561,"ariaHidden":333},[332],[296,3563,3565,3568,3571,3574,3623,3626,3666,3669,3672,3712],{"className":3564},[337],[296,3566],{"className":3567,"style":491},[341],[296,3569,1671],{"className":3570,"style":1632},[346,350],[296,3572,426],{"className":3573},[499],[296,3575,3577,3580],{"className":3576},[346],[296,3578,1422],{"className":3579},[346,350],[296,3581,3583],{"className":3582},[355],[296,3584,3586,3615],{"className":3585},[359,360],[296,3587,3589,3612],{"className":3588},[364],[296,3590,3592],{"className":3591,"style":1513},[368],[296,3593,3594,3597],{"style":1005},[296,3595],{"className":3596,"style":377},[376],[296,3598,3600],{"className":3599},[381,382,383,384],[296,3601,3603,3606,3609],{"className":3602},[346,384],[296,3604,1711],{"className":3605},[346,350,384],[296,3607,776],{"className":3608},[877,384],[296,3610,780],{"className":3611},[346,384],[296,3613,396],{"className":3614},[395],[296,3616,3618],{"className":3617},[364],[296,3619,3621],{"className":3620,"style":890},[368],[296,3622],{},[296,3624,820],{"className":3625},[346],[296,3627,3629,3632],{"className":3628},[346],[296,3630,1422],{"className":3631},[346,350],[296,3633,3635],{"className":3634},[355],[296,3636,3638,3658],{"className":3637},[359,360],[296,3639,3641,3655],{"className":3640},[364],[296,3642,3644],{"className":3643,"style":2065},[368],[296,3645,3646,3649],{"style":1005},[296,3647],{"className":3648,"style":377},[376],[296,3650,3652],{"className":3651},[381,382,383,384],[296,3653,1711],{"className":3654},[346,350,384],[296,3656,396],{"className":3657},[395],[296,3659,3661],{"className":3660},[364],[296,3662,3664],{"className":3663,"style":403},[368],[296,3665],{},[296,3667,1428],{"className":3668},[1539],[296,3670],{"className":3671,"style":973},[550],[296,3673,3675,3678],{"className":3674},[346],[296,3676,216],{"className":3677},[346,350],[296,3679,3681],{"className":3680},[355],[296,3682,3684,3704],{"className":3683},[359,360],[296,3685,3687,3701],{"className":3686},[364],[296,3688,3690],{"className":3689,"style":2065},[368],[296,3691,3692,3695],{"style":1005},[296,3693],{"className":3694,"style":377},[376],[296,3696,3698],{"className":3697},[381,382,383,384],[296,3699,1711],{"className":3700},[346,350,384],[296,3702,396],{"className":3703},[395],[296,3705,3707],{"className":3706},[364],[296,3708,3710],{"className":3709,"style":403},[368],[296,3711],{},[296,3713,435],{"className":3714},[546]," 和 ",[296,3717,3719,3738],{"className":3718},[299],[296,3720,3722],{"className":3721},[303],[305,3723,3724],{"xmlns":307},[309,3725,3726,3736],{},[312,3727,3728,3730,3732,3734],{},[318,3729,469],{},[423,3731,426],{"stretchy":425},[318,3733,454],{},[423,3735,435],{"stretchy":425},[325,3737,717],{"encoding":327},[296,3739,3741],{"className":3740,"ariaHidden":333},[332],[296,3742,3744,3747,3750,3753,3756],{"className":3743},[337],[296,3745],{"className":3746,"style":491},[341],[296,3748,469],{"className":3749,"style":683},[346,350],[296,3751,426],{"className":3752},[499],[296,3754,454],{"className":3755,"style":600},[346,350],[296,3757,435],{"className":3758},[546]," 的梯度都是零。",[212,3761,3762,3765],{},[247,3763,3764],{},"5. 轨迹的 grad-log-prob","。于是轨迹对数概率的梯度为",[296,3767,3769],{"className":3768},[751],[296,3770,3772,4033],{"className":3771},[299],[296,3773,3775],{"className":3774},[303],[305,3776,3777],{"xmlns":307,"display":760},[309,3778,3779,4030],{},[3780,3781,3785,3964],"mtable",{"rowspacing":3782,"columnalign":3783,"columnspacing":3784},"0.25em","right left","0em",[3786,3787,3788,3818],"mtr",{},[3789,3790,3791],"mtd",{},[3792,3793,3794],"mstyle",{"scriptlevel":1425,"displaystyle":333},[312,3795,3796,3802,3804,3806,3808,3810,3812,3814,3816],{},[315,3797,3798,3800],{},[318,3799,803],{"mathvariant":802},[318,3801,323],{},[318,3803,2289],{},[423,3805,2292],{},[318,3807,1671],{},[423,3809,426],{"stretchy":425},[318,3811,454],{},[318,3813,820],{"mathvariant":802},[318,3815,323],{},[423,3817,435],{"stretchy":425},[3789,3819,3820],{},[3792,3821,3822],{"scriptlevel":1425,"displaystyle":333},[312,3823,3824,3826,3828,3860,3862,3876,3878,3926,3928,3934,3936,3938,3944,3946,3952,3954,3960,3962],{},[312,3825],{},[423,3827,438],{},[3829,3830,3832],"menclose",{"notation":3831},"updiagonalstrike",[312,3833,3834,3840,3842,3844,3850,3852,3858],{},[315,3835,3836,3838],{},[318,3837,803],{"mathvariant":802},[318,3839,323],{},[318,3841,2289],{},[423,3843,2292],{},[315,3845,3846,3848],{},[318,3847,1688],{},[778,3849,1425],{},[423,3851,426],{"stretchy":425},[315,3853,3854,3856],{},[318,3855,1422],{},[778,3857,1425],{},[423,3859,435],{"stretchy":425},[423,3861,776],{},[1702,3863,3864,3866,3874],{},[423,3865,2704],{},[312,3867,3868,3870,3872],{},[318,3869,1711],{},[423,3871,438],{},[778,3873,1425],{},[318,3875,1454],{},[423,3877,426],{"fence":425,"stretchy":333,"minsize":2717,"maxsize":2717},[3829,3879,3880],{"notation":3831},[312,3881,3882,3888,3890,3892,3894,3896,3908,3910,3916,3918,3924],{},[315,3883,3884,3886],{},[318,3885,803],{"mathvariant":802},[318,3887,323],{},[318,3889,2289],{},[423,3891,2292],{},[318,3893,1671],{},[423,3895,426],{"stretchy":425},[315,3897,3898,3900],{},[318,3899,1422],{},[312,3901,3902,3904,3906],{},[318,3903,1711],{},[423,3905,776],{},[778,3907,780],{},[318,3909,820],{"mathvariant":802},[315,3911,3912,3914],{},[318,3913,1422],{},[318,3915,1711],{},[423,3917,1428],{"separator":333},[315,3919,3920,3922],{},[318,3921,216],{},[318,3923,1711],{},[423,3925,435],{"stretchy":425},[423,3927,776],{},[315,3929,3930,3932],{},[318,3931,803],{"mathvariant":802},[318,3933,323],{},[318,3935,2289],{},[423,3937,2292],{},[315,3939,3940,3942],{},[318,3941,320],{},[318,3943,323],{},[423,3945,426],{"stretchy":425},[315,3947,3948,3950],{},[318,3949,216],{},[318,3951,1711],{},[318,3953,820],{"mathvariant":802},[315,3955,3956,3958],{},[318,3957,1422],{},[318,3959,1711],{},[423,3961,435],{"stretchy":425},[423,3963,435],{"fence":425,"stretchy":333,"minsize":2717,"maxsize":2717},[3786,3965,3966,3972],{},[3789,3967,3968],{},[3792,3969,3970],{"scriptlevel":1425,"displaystyle":333},[312,3971],{},[3789,3973,3974],{},[3792,3975,3976],{"scriptlevel":1425,"displaystyle":333},[312,3977,3978,3980,3982,3996,4002,4004,4006,4012,4014,4020,4022,4028],{},[312,3979],{},[423,3981,438],{},[1702,3983,3984,3986,3994],{},[423,3985,2704],{},[312,3987,3988,3990,3992],{},[318,3989,1711],{},[423,3991,438],{},[778,3993,1425],{},[318,3995,1454],{},[315,3997,3998,4000],{},[318,3999,803],{"mathvariant":802},[318,4001,323],{},[318,4003,2289],{},[423,4005,2292],{},[315,4007,4008,4010],{},[318,4009,320],{},[318,4011,323],{},[423,4013,426],{"stretchy":425},[315,4015,4016,4018],{},[318,4017,216],{},[318,4019,1711],{},[318,4021,820],{"mathvariant":802},[315,4023,4024,4026],{},[318,4025,1422],{},[318,4027,1711],{},[423,4029,435],{"stretchy":425},[325,4031,4032],{"encoding":327},"\\begin{aligned}\n\\nabla_{\\theta} \\log P(\\tau | \\theta) &= \\cancel{\\nabla_{\\theta} \\log \\rho_0 (s_0)} + \\sum_{t=0}^{T} \\bigg( \\cancel{\\nabla_{\\theta} \\log P(s_{t+1}|s_t, a_t)}  + \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t)\\bigg) \\\\\n&= \\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t)\n\\end{aligned}",[296,4034,4036],{"className":4035,"ariaHidden":333},[332],[296,4037,4039,4043],{"className":4038},[337],[296,4040],{"className":4041,"style":4042},[341],"height:6.4909em;vertical-align:-2.9954em;",[296,4044,4046],{"className":4045},[346],[296,4047,4049,4166],{"className":4048},[3780],[296,4050,4053],{"className":4051},[4052],"col-align-r",[296,4054,4056,4157],{"className":4055},[359,360],[296,4057,4059,4154],{"className":4058},[364],[296,4060,4063,4145],{"className":4061,"style":4062},[368],"height:3.4954em;",[296,4064,4066,4070],{"style":4065},"top:-5.4954em;",[296,4067],{"className":4068,"style":4069},[376],"height:3.8283em;",[296,4071,4073,4116,4119,4124,4127,4130,4133,4136,4139,4142],{"className":4072},[346],[296,4074,4076,4079],{"className":4075},[346],[296,4077,803],{"className":4078},[346],[296,4080,4082],{"className":4081},[355],[296,4083,4085,4108],{"className":4084},[359,360],[296,4086,4088,4105],{"className":4087},[364],[296,4089,4091],{"className":4090,"style":369},[368],[296,4092,4093,4096],{"style":1005},[296,4094],{"className":4095,"style":377},[376],[296,4097,4099],{"className":4098},[381,382,383,384],[296,4100,4102],{"className":4101},[346,384],[296,4103,323],{"className":4104,"style":391},[346,350,384],[296,4106,396],{"className":4107},[395],[296,4109,4111],{"className":4110},[364],[296,4112,4114],{"className":4113,"style":403},[368],[296,4115],{},[296,4117],{"className":4118,"style":973},[550],[296,4120,2312,4122],{"className":4121},[572],[296,4123,2316],{"style":2315},[296,4125],{"className":4126,"style":973},[550],[296,4128,1671],{"className":4129,"style":1632},[346,350],[296,4131,426],{"className":4132},[499],[296,4134,454],{"className":4135,"style":600},[346,350],[296,4137,820],{"className":4138},[346],[296,4140,323],{"className":4141,"style":391},[346,350],[296,4143,435],{"className":4144},[546],[296,4146,4148,4151],{"style":4147},"top:-2.1em;",[296,4149],{"className":4150,"style":4069},[376],[296,4152],{"className":4153},[346],[296,4155,396],{"className":4156},[395],[296,4158,4160],{"className":4159},[364],[296,4161,4164],{"className":4162,"style":4163},[368],"height:2.9954em;",[296,4165],{},[296,4167,4170],{"className":4168},[4169],"col-align-l",[296,4171,4173,5204],{"className":4172},[359,360],[296,4174,4176,5201],{"className":4175},[364],[296,4177,4179,4925],{"className":4178,"style":4062},[368],[296,4180,4181,4184],{"style":4065},[296,4182],{"className":4183,"style":4069},[376],[296,4185,4187,4190,4193,4196,4199,4395,4398,4401,4404,4471,4474,4480,4724,4727,4730,4733,4776,4779,4784,4787,4830,4833,4873,4876,4916,4919],{"className":4186},[346],[296,4188],{"className":4189},[346],[296,4191],{"className":4192,"style":551},[550],[296,4194,438],{"className":4195},[555],[296,4197],{"className":4198,"style":551},[550],[296,4200,4203],{"className":4201},[346,4202],"cancel-lap",[296,4204,4206,4386],{"className":4205},[359,360],[296,4207,4209,4383],{"className":4208},[364],[296,4210,4213,4362],{"className":4211,"style":4212},[368],"height:0.75em;",[296,4214,4215,4218],{"style":655},[296,4216],{"className":4217,"style":590},[376],[296,4219,4222,4265,4268,4273,4276,4316,4319,4359],{"className":4220},[346,4221],"cancel-pad",[296,4223,4225,4228],{"className":4224},[346],[296,4226,803],{"className":4227},[346],[296,4229,4231],{"className":4230},[355],[296,4232,4234,4257],{"className":4233},[359,360],[296,4235,4237,4254],{"className":4236},[364],[296,4238,4240],{"className":4239,"style":369},[368],[296,4241,4242,4245],{"style":1005},[296,4243],{"className":4244,"style":377},[376],[296,4246,4248],{"className":4247},[381,382,383,384],[296,4249,4251],{"className":4250},[346,384],[296,4252,323],{"className":4253,"style":391},[346,350,384],[296,4255,396],{"className":4256},[395],[296,4258,4260],{"className":4259},[364],[296,4261,4263],{"className":4262,"style":403},[368],[296,4264],{},[296,4266],{"className":4267,"style":973},[550],[296,4269,2312,4271],{"className":4270},[572],[296,4272,2316],{"style":2315},[296,4274],{"className":4275,"style":973},[550],[296,4277,4279,4282],{"className":4278},[346],[296,4280,1688],{"className":4281},[346,350],[296,4283,4285],{"className":4284},[355],[296,4286,4288,4308],{"className":4287},[359,360],[296,4289,4291,4305],{"className":4290},[364],[296,4292,4294],{"className":4293,"style":1513},[368],[296,4295,4296,4299],{"style":1005},[296,4297],{"className":4298,"style":377},[376],[296,4300,4302],{"className":4301},[381,382,383,384],[296,4303,1425],{"className":4304},[346,384],[296,4306,396],{"className":4307},[395],[296,4309,4311],{"className":4310},[364],[296,4312,4314],{"className":4313,"style":403},[368],[296,4315],{},[296,4317,426],{"className":4318},[499],[296,4320,4322,4325],{"className":4321},[346],[296,4323,1422],{"className":4324},[346,350],[296,4326,4328],{"className":4327},[355],[296,4329,4331,4351],{"className":4330},[359,360],[296,4332,4334,4348],{"className":4333},[364],[296,4335,4337],{"className":4336,"style":1513},[368],[296,4338,4339,4342],{"style":1005},[296,4340],{"className":4341,"style":377},[376],[296,4343,4345],{"className":4344},[381,382,383,384],[296,4346,1425],{"className":4347},[346,384],[296,4349,396],{"className":4350},[395],[296,4352,4354],{"className":4353},[364],[296,4355,4357],{"className":4356,"style":403},[368],[296,4358],{},[296,4360,435],{"className":4361},[546],[296,4363,4367,4370],{"className":4364,"style":4366},[4365],"svg-align","top:-2.75em;",[296,4368],{"className":4369,"style":590},[376],[296,4371,4373],{"style":4372},"height:1em;",[4374,4375,4379],"svg",{"xmlns":4376,"width":4377,"height":4378},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","100%","1em",[4380,4381],"line",{"x1":1425,"y1":4377,"x2":4377,"y2":1425,"strokeWidth":4382},"0.046em",[296,4384,396],{"className":4385},[395],[296,4387,4389],{"className":4388},[364],[296,4390,4393],{"className":4391,"style":4392},[368],"height:0.25em;",[296,4394],{},[296,4396],{"className":4397,"style":952},[550],[296,4399,776],{"className":4400},[877],[296,4402],{"className":4403,"style":952},[550],[296,4405,4407],{"className":4406},[572,573],[296,4408,4410,4463],{"className":4409},[359,360],[296,4411,4413,4460],{"className":4412},[364],[296,4414,4416,4436,4446],{"className":4415,"style":1923},[368],[296,4417,4418,4421],{"style":1926},[296,4419],{"className":4420,"style":1930},[376],[296,4422,4424],{"className":4423},[381,382,383,384],[296,4425,4427,4430,4433],{"className":4426},[346,384],[296,4428,1711],{"className":4429},[346,350,384],[296,4431,438],{"className":4432},[555,384],[296,4434,1425],{"className":4435},[346,384],[296,4437,4438,4441],{"style":1948},[296,4439],{"className":4440,"style":1930},[376],[296,4442,4443],{},[296,4444,2704],{"className":4445},[572,1957,1958],[296,4447,4448,4451],{"style":1961},[296,4449],{"className":4450,"style":1930},[376],[296,4452,4454],{"className":4453},[381,382,383,384],[296,4455,4457],{"className":4456},[346,384],[296,4458,1454],{"className":4459,"style":1632},[346,350,384],[296,4461,396],{"className":4462},[395],[296,4464,4466],{"className":4465},[364],[296,4467,4469],{"className":4468,"style":1983},[368],[296,4470],{},[296,4472],{"className":4473,"style":973},[550],[296,4475,4477],{"className":4476},[346],[296,4478,426],{"className":4479},[3026,383],[296,4481,4483],{"className":4482},[346,4202],[296,4484,4486,4716],{"className":4485},[359,360],[296,4487,4489,4713],{"className":4488},[364],[296,4490,4492,4701],{"className":4491,"style":4212},[368],[296,4493,4494,4497],{"style":655},[296,4495],{"className":4496,"style":590},[376],[296,4498,4500,4543,4546,4551,4554,4557,4560,4609,4612,4652,4655,4658,4698],{"className":4499},[346,4221],[296,4501,4503,4506],{"className":4502},[346],[296,4504,803],{"className":4505},[346],[296,4507,4509],{"className":4508},[355],[296,4510,4512,4535],{"className":4511},[359,360],[296,4513,4515,4532],{"className":4514},[364],[296,4516,4518],{"className":4517,"style":369},[368],[296,4519,4520,4523],{"style":1005},[296,4521],{"className":4522,"style":377},[376],[296,4524,4526],{"className":4525},[381,382,383,384],[296,4527,4529],{"className":4528},[346,384],[296,4530,323],{"className":4531,"style":391},[346,350,384],[296,4533,396],{"className":4534},[395],[296,4536,4538],{"className":4537},[364],[296,4539,4541],{"className":4540,"style":403},[368],[296,4542],{},[296,4544],{"className":4545,"style":973},[550],[296,4547,2312,4549],{"className":4548},[572],[296,4550,2316],{"style":2315},[296,4552],{"className":4553,"style":973},[550],[296,4555,1671],{"className":4556,"style":1632},[346,350],[296,4558,426],{"className":4559},[499],[296,4561,4563,4566],{"className":4562},[346],[296,4564,1422],{"className":4565},[346,350],[296,4567,4569],{"className":4568},[355],[296,4570,4572,4601],{"className":4571},[359,360],[296,4573,4575,4598],{"className":4574},[364],[296,4576,4578],{"className":4577,"style":1513},[368],[296,4579,4580,4583],{"style":1005},[296,4581],{"className":4582,"style":377},[376],[296,4584,4586],{"className":4585},[381,382,383,384],[296,4587,4589,4592,4595],{"className":4588},[346,384],[296,4590,1711],{"className":4591},[346,350,384],[296,4593,776],{"className":4594},[877,384],[296,4596,780],{"className":4597},[346,384],[296,4599,396],{"className":4600},[395],[296,4602,4604],{"className":4603},[364],[296,4605,4607],{"className":4606,"style":890},[368],[296,4608],{},[296,4610,820],{"className":4611},[346],[296,4613,4615,4618],{"className":4614},[346],[296,4616,1422],{"className":4617},[346,350],[296,4619,4621],{"className":4620},[355],[296,4622,4624,4644],{"className":4623},[359,360],[296,4625,4627,4641],{"className":4626},[364],[296,4628,4630],{"className":4629,"style":2065},[368],[296,4631,4632,4635],{"style":1005},[296,4633],{"className":4634,"style":377},[376],[296,4636,4638],{"className":4637},[381,382,383,384],[296,4639,1711],{"className":4640},[346,350,384],[296,4642,396],{"className":4643},[395],[296,4645,4647],{"className":4646},[364],[296,4648,4650],{"className":4649,"style":403},[368],[296,4651],{},[296,4653,1428],{"className":4654},[1539],[296,4656],{"className":4657,"style":973},[550],[296,4659,4661,4664],{"className":4660},[346],[296,4662,216],{"className":4663},[346,350],[296,4665,4667],{"className":4666},[355],[296,4668,4670,4690],{"className":4669},[359,360],[296,4671,4673,4687],{"className":4672},[364],[296,4674,4676],{"className":4675,"style":2065},[368],[296,4677,4678,4681],{"style":1005},[296,4679],{"className":4680,"style":377},[376],[296,4682,4684],{"className":4683},[381,382,383,384],[296,4685,1711],{"className":4686},[346,350,384],[296,4688,396],{"className":4689},[395],[296,4691,4693],{"className":4692},[364],[296,4694,4696],{"className":4695,"style":403},[368],[296,4697],{},[296,4699,435],{"className":4700},[546],[296,4702,4704,4707],{"className":4703,"style":4366},[4365],[296,4705],{"className":4706,"style":590},[376],[296,4708,4709],{"style":4372},[4374,4710,4711],{"xmlns":4376,"width":4377,"height":4378},[4380,4712],{"x1":1425,"y1":4377,"x2":4377,"y2":1425,"strokeWidth":4382},[296,4714,396],{"className":4715},[395],[296,4717,4719],{"className":4718},[364],[296,4720,4722],{"className":4721,"style":4392},[368],[296,4723],{},[296,4725],{"className":4726,"style":952},[550],[296,4728,776],{"className":4729},[877],[296,4731],{"className":4732,"style":952},[550],[296,4734,4736,4739],{"className":4735},[346],[296,4737,803],{"className":4738},[346],[296,4740,4742],{"className":4741},[355],[296,4743,4745,4768],{"className":4744},[359,360],[296,4746,4748,4765],{"className":4747},[364],[296,4749,4751],{"className":4750,"style":369},[368],[296,4752,4753,4756],{"style":1005},[296,4754],{"className":4755,"style":377},[376],[296,4757,4759],{"className":4758},[381,382,383,384],[296,4760,4762],{"className":4761},[346,384],[296,4763,323],{"className":4764,"style":391},[346,350,384],[296,4766,396],{"className":4767},[395],[296,4769,4771],{"className":4770},[364],[296,4772,4774],{"className":4773,"style":403},[368],[296,4775],{},[296,4777],{"className":4778,"style":973},[550],[296,4780,2312,4782],{"className":4781},[572],[296,4783,2316],{"style":2315},[296,4785],{"className":4786,"style":973},[550],[296,4788,4790,4793],{"className":4789},[346],[296,4791,320],{"className":4792,"style":351},[346,350],[296,4794,4796],{"className":4795},[355],[296,4797,4799,4822],{"className":4798},[359,360],[296,4800,4802,4819],{"className":4801},[364],[296,4803,4805],{"className":4804,"style":369},[368],[296,4806,4807,4810],{"style":372},[296,4808],{"className":4809,"style":377},[376],[296,4811,4813],{"className":4812},[381,382,383,384],[296,4814,4816],{"className":4815},[346,384],[296,4817,323],{"className":4818,"style":391},[346,350,384],[296,4820,396],{"className":4821},[395],[296,4823,4825],{"className":4824},[364],[296,4826,4828],{"className":4827,"style":403},[368],[296,4829],{},[296,4831,426],{"className":4832},[499],[296,4834,4836,4839],{"className":4835},[346],[296,4837,216],{"className":4838},[346,350],[296,4840,4842],{"className":4841},[355],[296,4843,4845,4865],{"className":4844},[359,360],[296,4846,4848,4862],{"className":4847},[364],[296,4849,4851],{"className":4850,"style":2065},[368],[296,4852,4853,4856],{"style":1005},[296,4854],{"className":4855,"style":377},[376],[296,4857,4859],{"className":4858},[381,382,383,384],[296,4860,1711],{"className":4861},[346,350,384],[296,4863,396],{"className":4864},[395],[296,4866,4868],{"className":4867},[364],[296,4869,4871],{"className":4870,"style":403},[368],[296,4872],{},[296,4874,820],{"className":4875},[346],[296,4877,4879,4882],{"className":4878},[346],[296,4880,1422],{"className":4881},[346,350],[296,4883,4885],{"className":4884},[355],[296,4886,4888,4908],{"className":4887},[359,360],[296,4889,4891,4905],{"className":4890},[364],[296,4892,4894],{"className":4893,"style":2065},[368],[296,4895,4896,4899],{"style":1005},[296,4897],{"className":4898,"style":377},[376],[296,4900,4902],{"className":4901},[381,382,383,384],[296,4903,1711],{"className":4904},[346,350,384],[296,4906,396],{"className":4907},[395],[296,4909,4911],{"className":4910},[364],[296,4912,4914],{"className":4913,"style":403},[368],[296,4915],{},[296,4917,435],{"className":4918},[546],[296,4920,4922],{"className":4921},[346],[296,4923,435],{"className":4924},[3026,383],[296,4926,4927,4930],{"style":4147},[296,4928],{"className":4929,"style":4069},[376],[296,4931,4933,4936,4939,4942,4945,5012,5015,5058,5061,5066,5069,5112,5115,5155,5158,5198],{"className":4932},[346],[296,4934],{"className":4935},[346],[296,4937],{"className":4938,"style":551},[550],[296,4940,438],{"className":4941},[555],[296,4943],{"className":4944,"style":551},[550],[296,4946,4948],{"className":4947},[572,573],[296,4949,4951,5004],{"className":4950},[359,360],[296,4952,4954,5001],{"className":4953},[364],[296,4955,4957,4977,4987],{"className":4956,"style":1923},[368],[296,4958,4959,4962],{"style":1926},[296,4960],{"className":4961,"style":1930},[376],[296,4963,4965],{"className":4964},[381,382,383,384],[296,4966,4968,4971,4974],{"className":4967},[346,384],[296,4969,1711],{"className":4970},[346,350,384],[296,4972,438],{"className":4973},[555,384],[296,4975,1425],{"className":4976},[346,384],[296,4978,4979,4982],{"style":1948},[296,4980],{"className":4981,"style":1930},[376],[296,4983,4984],{},[296,4985,2704],{"className":4986},[572,1957,1958],[296,4988,4989,4992],{"style":1961},[296,4990],{"className":4991,"style":1930},[376],[296,4993,4995],{"className":4994},[381,382,383,384],[296,4996,4998],{"className":4997},[346,384],[296,4999,1454],{"className":5000,"style":1632},[346,350,384],[296,5002,396],{"className":5003},[395],[296,5005,5007],{"className":5006},[364],[296,5008,5010],{"className":5009,"style":1983},[368],[296,5011],{},[296,5013],{"className":5014,"style":973},[550],[296,5016,5018,5021],{"className":5017},[346],[296,5019,803],{"className":5020},[346],[296,5022,5024],{"className":5023},[355],[296,5025,5027,5050],{"className":5026},[359,360],[296,5028,5030,5047],{"className":5029},[364],[296,5031,5033],{"className":5032,"style":369},[368],[296,5034,5035,5038],{"style":1005},[296,5036],{"className":5037,"style":377},[376],[296,5039,5041],{"className":5040},[381,382,383,384],[296,5042,5044],{"className":5043},[346,384],[296,5045,323],{"className":5046,"style":391},[346,350,384],[296,5048,396],{"className":5049},[395],[296,5051,5053],{"className":5052},[364],[296,5054,5056],{"className":5055,"style":403},[368],[296,5057],{},[296,5059],{"className":5060,"style":973},[550],[296,5062,2312,5064],{"className":5063},[572],[296,5065,2316],{"style":2315},[296,5067],{"className":5068,"style":973},[550],[296,5070,5072,5075],{"className":5071},[346],[296,5073,320],{"className":5074,"style":351},[346,350],[296,5076,5078],{"className":5077},[355],[296,5079,5081,5104],{"className":5080},[359,360],[296,5082,5084,5101],{"className":5083},[364],[296,5085,5087],{"className":5086,"style":369},[368],[296,5088,5089,5092],{"style":372},[296,5090],{"className":5091,"style":377},[376],[296,5093,5095],{"className":5094},[381,382,383,384],[296,5096,5098],{"className":5097},[346,384],[296,5099,323],{"className":5100,"style":391},[346,350,384],[296,5102,396],{"className":5103},[395],[296,5105,5107],{"className":5106},[364],[296,5108,5110],{"className":5109,"style":403},[368],[296,5111],{},[296,5113,426],{"className":5114},[499],[296,5116,5118,5121],{"className":5117},[346],[296,5119,216],{"className":5120},[346,350],[296,5122,5124],{"className":5123},[355],[296,5125,5127,5147],{"className":5126},[359,360],[296,5128,5130,5144],{"className":5129},[364],[296,5131,5133],{"className":5132,"style":2065},[368],[296,5134,5135,5138],{"style":1005},[296,5136],{"className":5137,"style":377},[376],[296,5139,5141],{"className":5140},[381,382,383,384],[296,5142,1711],{"className":5143},[346,350,384],[296,5145,396],{"className":5146},[395],[296,5148,5150],{"className":5149},[364],[296,5151,5153],{"className":5152,"style":403},[368],[296,5154],{},[296,5156,820],{"className":5157},[346],[296,5159,5161,5164],{"className":5160},[346],[296,5162,1422],{"className":5163},[346,350],[296,5165,5167],{"className":5166},[355],[296,5168,5170,5190],{"className":5169},[359,360],[296,5171,5173,5187],{"className":5172},[364],[296,5174,5176],{"className":5175,"style":2065},[368],[296,5177,5178,5181],{"style":1005},[296,5179],{"className":5180,"style":377},[376],[296,5182,5184],{"className":5183},[381,382,383,384],[296,5185,1711],{"className":5186},[346,350,384],[296,5188,396],{"className":5189},[395],[296,5191,5193],{"className":5192},[364],[296,5194,5196],{"className":5195,"style":403},[368],[296,5197],{},[296,5199,435],{"className":5200},[546],[296,5202,396],{"className":5203},[395],[296,5205,5207],{"className":5206},[364],[296,5208,5210],{"className":5209,"style":4163},[368],[296,5211],{},[212,5213,5214],{},"把这些拼起来，推出：",[208,5216,5218],{"title":5217},"基本策略梯度的推导",[296,5219,5221],{"className":5220},[751],[296,5222,5224,5772],{"className":5223},[299],[296,5225,5227],{"className":5226},[303],[305,5228,5229],{"xmlns":307,"display":760},[309,5230,5231,5769],{},[3780,5232,5235,5321,5390,5457,5540,5631],{"rowspacing":3782,"columnalign":5233,"columnspacing":5234},"right left right left","0em 1em 0em",[3786,5236,5237,5261,5309,5315],{},[3789,5238,5239],{},[3792,5240,5241],{"scriptlevel":1425,"displaystyle":333},[312,5242,5243,5249,5251,5253,5259],{},[315,5244,5245,5247],{},[318,5246,803],{"mathvariant":802},[318,5248,323],{},[318,5250,421],{},[423,5252,426],{"stretchy":425},[315,5254,5255,5257],{},[318,5256,320],{},[318,5258,323],{},[423,5260,435],{"stretchy":425},[3789,5262,5263],{},[3792,5264,5265],{"scriptlevel":1425,"displaystyle":333},[312,5266,5267,5269,5271,5277,5297,5299,5301,5303,5305,5307],{},[312,5268],{},[423,5270,438],{},[315,5272,5273,5275],{},[318,5274,803],{"mathvariant":802},[318,5276,323],{},[318,5278,5279],{},[442,5280,5281,5285],{},[423,5282,5283],{},[318,5284,449],{"mathvariant":448},[312,5286,5287,5289,5291],{},[318,5288,454],{},[423,5290,457],{},[315,5292,5293,5295],{},[318,5294,320],{},[318,5296,323],{},[423,5298,466],{"stretchy":425},[318,5300,469],{},[423,5302,426],{"stretchy":425},[318,5304,454],{},[423,5306,435],{"stretchy":425},[423,5308,478],{"stretchy":425},[3789,5310,5311],{},[3792,5312,5313],{"scriptlevel":1425,"displaystyle":333},[312,5314],{},[3789,5316,5317],{},[3792,5318,5319],{"scriptlevel":1425,"displaystyle":333},[312,5320],{},[3786,5322,5323,5329,5372,5378],{},[3789,5324,5325],{},[3792,5326,5327],{"scriptlevel":1425,"displaystyle":333},[312,5328],{},[3789,5330,5331],{},[3792,5332,5333],{"scriptlevel":1425,"displaystyle":333},[312,5334,5335,5337,5339,5345,5352,5354,5356,5358,5360,5362,5364,5366,5368,5370],{},[312,5336],{},[423,5338,438],{},[315,5340,5341,5343],{},[318,5342,803],{"mathvariant":802},[318,5344,323],{},[315,5346,5347,5350],{},[423,5348,5349],{},"∫",[318,5351,454],{},[318,5353,1671],{},[423,5355,426],{"stretchy":425},[318,5357,454],{},[318,5359,820],{"mathvariant":802},[318,5361,323],{},[423,5363,435],{"stretchy":425},[318,5365,469],{},[423,5367,426],{"stretchy":425},[318,5369,454],{},[423,5371,435],{"stretchy":425},[3789,5373,5374],{},[3792,5375,5376],{"scriptlevel":1425,"displaystyle":333},[312,5377],{},[3789,5379,5380],{},[3792,5381,5382],{"scriptlevel":1425,"displaystyle":333},[312,5383,5384,5386],{},[312,5385],{},[5387,5388,5389],"mtext",{},"展开期望",[3786,5391,5392,5398,5440,5446],{},[3789,5393,5394],{},[3792,5395,5396],{"scriptlevel":1425,"displaystyle":333},[312,5397],{},[3789,5399,5400],{},[3792,5401,5402],{"scriptlevel":1425,"displaystyle":333},[312,5403,5404,5406,5408,5414,5420,5422,5424,5426,5428,5430,5432,5434,5436,5438],{},[312,5405],{},[423,5407,438],{},[315,5409,5410,5412],{},[423,5411,5349],{},[318,5413,454],{},[315,5415,5416,5418],{},[318,5417,803],{"mathvariant":802},[318,5419,323],{},[318,5421,1671],{},[423,5423,426],{"stretchy":425},[318,5425,454],{},[318,5427,820],{"mathvariant":802},[318,5429,323],{},[423,5431,435],{"stretchy":425},[318,5433,469],{},[423,5435,426],{"stretchy":425},[318,5437,454],{},[423,5439,435],{"stretchy":425},[3789,5441,5442],{},[3792,5443,5444],{"scriptlevel":1425,"displaystyle":333},[312,5445],{},[3789,5447,5448],{},[3792,5449,5450],{"scriptlevel":1425,"displaystyle":333},[312,5451,5452,5454],{},[312,5453],{},[5387,5455,5456],{},"把梯度移入积分号内",[3786,5458,5459,5465,5523,5529],{},[3789,5460,5461],{},[3792,5462,5463],{"scriptlevel":1425,"displaystyle":333},[312,5464],{},[3789,5466,5467],{},[3792,5468,5469],{"scriptlevel":1425,"displaystyle":333},[312,5470,5471,5473,5475,5481,5483,5485,5487,5489,5491,5493,5499,5501,5503,5505,5507,5509,5511,5513,5515,5517,5519,5521],{},[312,5472],{},[423,5474,438],{},[315,5476,5477,5479],{},[423,5478,5349],{},[318,5480,454],{},[318,5482,1671],{},[423,5484,426],{"stretchy":425},[318,5486,454],{},[318,5488,820],{"mathvariant":802},[318,5490,323],{},[423,5492,435],{"stretchy":425},[315,5494,5495,5497],{},[318,5496,803],{"mathvariant":802},[318,5498,323],{},[318,5500,2289],{},[423,5502,2292],{},[318,5504,1671],{},[423,5506,426],{"stretchy":425},[318,5508,454],{},[318,5510,820],{"mathvariant":802},[318,5512,323],{},[423,5514,435],{"stretchy":425},[318,5516,469],{},[423,5518,426],{"stretchy":425},[318,5520,454],{},[423,5522,435],{"stretchy":425},[3789,5524,5525],{},[3792,5526,5527],{"scriptlevel":1425,"displaystyle":333},[312,5528],{},[3789,5530,5531],{},[3792,5532,5533],{"scriptlevel":1425,"displaystyle":333},[312,5534,5535,5537],{},[312,5536],{},[5387,5538,5539],{},"对数求导技巧",[3786,5541,5542,5548,5614,5620],{},[3789,5543,5544],{},[3792,5545,5546],{"scriptlevel":1425,"displaystyle":333},[312,5547],{},[3789,5549,5550],{},[3792,5551,5552],{"scriptlevel":1425,"displaystyle":333},[312,5553,5554,5556,5558,5578],{},[312,5555],{},[423,5557,438],{},[318,5559,5560],{},[442,5561,5562,5566],{},[423,5563,5564],{},[318,5565,449],{"mathvariant":448},[312,5567,5568,5570,5572],{},[318,5569,454],{},[423,5571,457],{},[315,5573,5574,5576],{},[318,5575,320],{},[318,5577,323],{},[312,5579,5580,5582,5588,5590,5592,5594,5596,5598,5600,5602,5604,5606,5608,5610,5612],{},[423,5581,466],{"fence":333},[315,5583,5584,5586],{},[318,5585,803],{"mathvariant":802},[318,5587,323],{},[318,5589,2289],{},[423,5591,2292],{},[318,5593,1671],{},[423,5595,426],{"stretchy":425},[318,5597,454],{},[318,5599,820],{"mathvariant":802},[318,5601,323],{},[423,5603,435],{"stretchy":425},[318,5605,469],{},[423,5607,426],{"stretchy":425},[318,5609,454],{},[423,5611,435],{"stretchy":425},[423,5613,478],{"fence":333},[3789,5615,5616],{},[3792,5617,5618],{"scriptlevel":1425,"displaystyle":333},[312,5619],{},[3789,5621,5622],{},[3792,5623,5624],{"scriptlevel":1425,"displaystyle":333},[312,5625,5626,5628],{},[312,5627],{},[5387,5629,5630],{},"回到期望形式",[3786,5632,5633,5660,5752,5758],{},[3789,5634,5635],{},[3792,5636,5637],{"scriptlevel":1425,"displaystyle":333},[312,5638,5639,5642,5648,5650,5652,5658],{},[423,5640,5641],{},"∴",[315,5643,5644,5646],{},[318,5645,803],{"mathvariant":802},[318,5647,323],{},[318,5649,421],{},[423,5651,426],{"stretchy":425},[315,5653,5654,5656],{},[318,5655,320],{},[318,5657,323],{},[423,5659,435],{"stretchy":425},[3789,5661,5662],{},[3792,5663,5664],{"scriptlevel":1425,"displaystyle":333},[312,5665,5666,5668,5670,5690],{},[312,5667],{},[423,5669,438],{},[318,5671,5672],{},[442,5673,5674,5678],{},[423,5675,5676],{},[318,5677,449],{"mathvariant":448},[312,5679,5680,5682,5684],{},[318,5681,454],{},[423,5683,457],{},[315,5685,5686,5688],{},[318,5687,320],{},[318,5689,323],{},[312,5691,5692,5694,5708,5714,5716,5718,5724,5726,5732,5734,5740,5742,5744,5746,5748,5750],{},[423,5693,466],{"fence":333},[1702,5695,5696,5698,5706],{},[423,5697,2704],{},[312,5699,5700,5702,5704],{},[318,5701,1711],{},[423,5703,438],{},[778,5705,1425],{},[318,5707,1454],{},[315,5709,5710,5712],{},[318,5711,803],{"mathvariant":802},[318,5713,323],{},[318,5715,2289],{},[423,5717,2292],{},[315,5719,5720,5722],{},[318,5721,320],{},[318,5723,323],{},[423,5725,426],{"stretchy":425},[315,5727,5728,5730],{},[318,5729,216],{},[318,5731,1711],{},[318,5733,820],{"mathvariant":802},[315,5735,5736,5738],{},[318,5737,1422],{},[318,5739,1711],{},[423,5741,435],{"stretchy":425},[318,5743,469],{},[423,5745,426],{"stretchy":425},[318,5747,454],{},[423,5749,435],{"stretchy":425},[423,5751,478],{"fence":333},[3789,5753,5754],{},[3792,5755,5756],{"scriptlevel":1425,"displaystyle":333},[312,5757],{},[3789,5759,5760],{},[3792,5761,5762],{"scriptlevel":1425,"displaystyle":333},[312,5763,5764,5766],{},[312,5765],{},[5387,5767,5768],{},"代入轨迹的 grad-log-prob",[325,5770,5771],{"encoding":327},"\\begin{align*}\n\\nabla_{\\theta} J(\\pi_{\\theta}) &= \\nabla_{\\theta} \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}[R(\\tau)] && \\\\\n&= \\nabla_{\\theta} \\int_{\\tau} P(\\tau|\\theta) R(\\tau) && \\text{展开期望} \\\\\n&= \\int_{\\tau} \\nabla_{\\theta} P(\\tau|\\theta) R(\\tau) && \\text{把梯度移入积分号内} \\\\\n&= \\int_{\\tau} P(\\tau|\\theta) \\nabla_{\\theta} \\log P(\\tau|\\theta) R(\\tau) && \\text{对数求导技巧} \\\\\n&= \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\nabla_{\\theta} \\log P(\\tau|\\theta) R(\\tau)\\right] && \\text{回到期望形式} \\\\\n\\therefore \\nabla_{\\theta} J(\\pi_{\\theta}) &= \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) R(\\tau)\\right] && \\text{代入轨迹的 grad-log-prob}\n\\end{align*}",[296,5773,5775],{"className":5774,"ariaHidden":333},[332],[296,5776,5778,5782],{"className":5777},[337],[296,5779],{"className":5780,"style":5781},[341],"height:14.703em;vertical-align:-7.1015em;",[296,5783,5785],{"className":5784},[346],[296,5786,5788,6064,7329,7334,7405],{"className":5787},[3780],[296,5789,5791],{"className":5790},[4052],[296,5792,5794,6055],{"className":5793},[359,360],[296,5795,5797,6052],{"className":5796},[364],[296,5798,5801,5905,5914,5923,5932,5941],{"className":5799,"style":5800},[368],"height:7.6015em;",[296,5802,5804,5807],{"style":5803},"top:-10.5898em;",[296,5805],{"className":5806,"style":4069},[376],[296,5808,5810,5853,5856,5859,5902],{"className":5809},[346],[296,5811,5813,5816],{"className":5812},[346],[296,5814,803],{"className":5815},[346],[296,5817,5819],{"className":5818},[355],[296,5820,5822,5845],{"className":5821},[359,360],[296,5823,5825,5842],{"className":5824},[364],[296,5826,5828],{"className":5827,"style":369},[368],[296,5829,5830,5833],{"style":1005},[296,5831],{"className":5832,"style":377},[376],[296,5834,5836],{"className":5835},[381,382,383,384],[296,5837,5839],{"className":5838},[346,384],[296,5840,323],{"className":5841,"style":391},[346,350,384],[296,5843,396],{"className":5844},[395],[296,5846,5848],{"className":5847},[364],[296,5849,5851],{"className":5850,"style":403},[368],[296,5852],{},[296,5854,421],{"className":5855,"style":495},[346,350],[296,5857,426],{"className":5858},[499],[296,5860,5862,5865],{"className":5861},[346],[296,5863,320],{"className":5864,"style":351},[346,350],[296,5866,5868],{"className":5867},[355],[296,5869,5871,5894],{"className":5870},[359,360],[296,5872,5874,5891],{"className":5873},[364],[296,5875,5877],{"className":5876,"style":369},[368],[296,5878,5879,5882],{"style":372},[296,5880],{"className":5881,"style":377},[376],[296,5883,5885],{"className":5884},[381,382,383,384],[296,5886,5888],{"className":5887},[346,384],[296,5889,323],{"className":5890,"style":391},[346,350,384],[296,5892,396],{"className":5893},[395],[296,5895,5897],{"className":5896},[364],[296,5898,5900],{"className":5899,"style":403},[368],[296,5901],{},[296,5903,435],{"className":5904},[546],[296,5906,5908,5911],{"style":5907},"top:-8.124em;",[296,5909],{"className":5910,"style":4069},[376],[296,5912],{"className":5913},[346],[296,5915,5917,5920],{"style":5916},"top:-5.552em;",[296,5918],{"className":5919,"style":4069},[376],[296,5921],{"className":5922},[346],[296,5924,5926,5929],{"style":5925},"top:-2.9801em;",[296,5927],{"className":5928,"style":4069},[376],[296,5930],{"className":5931},[346],[296,5933,5935,5938],{"style":5934},"top:-0.9281em;",[296,5936],{"className":5937,"style":4069},[376],[296,5939],{"className":5940},[346],[296,5942,5944,5947],{"style":5943},"top:2.0061em;",[296,5945],{"className":5946,"style":4069},[376],[296,5948,5950,5954,5957,6000,6003,6006,6049],{"className":5949},[346],[296,5951,5641],{"className":5952},[555,5953],"amsrm",[296,5955],{"className":5956,"style":551},[550],[296,5958,5960,5963],{"className":5959},[346],[296,5961,803],{"className":5962},[346],[296,5964,5966],{"className":5965},[355],[296,5967,5969,5992],{"className":5968},[359,360],[296,5970,5972,5989],{"className":5971},[364],[296,5973,5975],{"className":5974,"style":369},[368],[296,5976,5977,5980],{"style":1005},[296,5978],{"className":5979,"style":377},[376],[296,5981,5983],{"className":5982},[381,382,383,384],[296,5984,5986],{"className":5985},[346,384],[296,5987,323],{"className":5988,"style":391},[346,350,384],[296,5990,396],{"className":5991},[395],[296,5993,5995],{"className":5994},[364],[296,5996,5998],{"className":5997,"style":403},[368],[296,5999],{},[296,6001,421],{"className":6002,"style":495},[346,350],[296,6004,426],{"className":6005},[499],[296,6007,6009,6012],{"className":6008},[346],[296,6010,320],{"className":6011,"style":351},[346,350],[296,6013,6015],{"className":6014},[355],[296,6016,6018,6041],{"className":6017},[359,360],[296,6019,6021,6038],{"className":6020},[364],[296,6022,6024],{"className":6023,"style":369},[368],[296,6025,6026,6029],{"style":372},[296,6027],{"className":6028,"style":377},[376],[296,6030,6032],{"className":6031},[381,382,383,384],[296,6033,6035],{"className":6034},[346,384],[296,6036,323],{"className":6037,"style":391},[346,350,384],[296,6039,396],{"className":6040},[395],[296,6042,6044],{"className":6043},[364],[296,6045,6047],{"className":6046,"style":403},[368],[296,6048],{},[296,6050,435],{"className":6051},[546],[296,6053,396],{"className":6054},[395],[296,6056,6058],{"className":6057},[364],[296,6059,6062],{"className":6060,"style":6061},[368],"height:7.1015em;",[296,6063],{},[296,6065,6067],{"className":6066},[4169],[296,6068,6070,7321],{"className":6069},[359,360],[296,6071,6073,7318],{"className":6072},[364],[296,6074,6076,6250,6396,6535,6703,6915],{"className":6075,"style":5800},[368],[296,6077,6078,6081],{"style":5803},[296,6079],{"className":6080,"style":4069},[376],[296,6082,6084,6087,6090,6093,6096,6139,6235,6238,6241,6244,6247],{"className":6083},[346],[296,6085],{"className":6086},[346],[296,6088],{"className":6089,"style":551},[550],[296,6091,438],{"className":6092},[555],[296,6094],{"className":6095,"style":551},[550],[296,6097,6099,6102],{"className":6098},[346],[296,6100,803],{"className":6101},[346],[296,6103,6105],{"className":6104},[355],[296,6106,6108,6131],{"className":6107},[359,360],[296,6109,6111,6128],{"className":6110},[364],[296,6112,6114],{"className":6113,"style":369},[368],[296,6115,6116,6119],{"style":1005},[296,6117],{"className":6118,"style":377},[376],[296,6120,6122],{"className":6121},[381,382,383,384],[296,6123,6125],{"className":6124},[346,384],[296,6126,323],{"className":6127,"style":391},[346,350,384],[296,6129,396],{"className":6130},[395],[296,6132,6134],{"className":6133},[364],[296,6135,6137],{"className":6136,"style":403},[368],[296,6138],{},[296,6140,6142],{"className":6141},[346],[296,6143,6145],{"className":6144},[572,573],[296,6146,6148,6227],{"className":6147},[359,360],[296,6149,6151,6224],{"className":6150},[364],[296,6152,6154,6214],{"className":6153,"style":583},[368],[296,6155,6156,6159],{"style":586},[296,6157],{"className":6158,"style":590},[376],[296,6160,6162],{"className":6161},[381,382,383,384],[296,6163,6165,6168,6171],{"className":6164},[346,384],[296,6166,454],{"className":6167,"style":600},[346,350,384],[296,6169,457],{"className":6170},[555,384],[296,6172,6174,6177],{"className":6173},[346,384],[296,6175,320],{"className":6176,"style":351},[346,350,384],[296,6178,6180],{"className":6179},[355],[296,6181,6183,6206],{"className":6182},[359,360],[296,6184,6186,6203],{"className":6185},[364],[296,6187,6189],{"className":6188,"style":622},[368],[296,6190,6191,6194],{"style":625},[296,6192],{"className":6193,"style":629},[376],[296,6195,6197],{"className":6196},[381,633,634,384],[296,6198,6200],{"className":6199},[346,384],[296,6201,323],{"className":6202,"style":391},[346,350,384],[296,6204,396],{"className":6205},[395],[296,6207,6209],{"className":6208},[364],[296,6210,6212],{"className":6211,"style":650},[368],[296,6213],{},[296,6215,6216,6219],{"style":655},[296,6217],{"className":6218,"style":590},[376],[296,6220,6221],{},[296,6222,449],{"className":6223},[572,664],[296,6225,396],{"className":6226},[395],[296,6228,6230],{"className":6229},[364],[296,6231,6233],{"className":6232,"style":674},[368],[296,6234],{},[296,6236,466],{"className":6237},[499],[296,6239,469],{"className":6240,"style":683},[346,350],[296,6242,426],{"className":6243},[499],[296,6245,454],{"className":6246,"style":600},[346,350],[296,6248,693],{"className":6249},[546],[296,6251,6252,6255],{"style":5907},[296,6253],{"className":6254,"style":4069},[376],[296,6256,6258,6261,6264,6267,6270,6313,6316,6363,6366,6369,6372,6375,6378,6381,6384,6387,6390,6393],{"className":6257},[346],[296,6259],{"className":6260},[346],[296,6262],{"className":6263,"style":551},[550],[296,6265,438],{"className":6266},[555],[296,6268],{"className":6269,"style":551},[550],[296,6271,6273,6276],{"className":6272},[346],[296,6274,803],{"className":6275},[346],[296,6277,6279],{"className":6278},[355],[296,6280,6282,6305],{"className":6281},[359,360],[296,6283,6285,6302],{"className":6284},[364],[296,6286,6288],{"className":6287,"style":369},[368],[296,6289,6290,6293],{"style":1005},[296,6291],{"className":6292,"style":377},[376],[296,6294,6296],{"className":6295},[381,382,383,384],[296,6297,6299],{"className":6298},[346,384],[296,6300,323],{"className":6301,"style":391},[346,350,384],[296,6303,396],{"className":6304},[395],[296,6306,6308],{"className":6307},[364],[296,6309,6311],{"className":6310,"style":403},[368],[296,6312],{},[296,6314],{"className":6315,"style":973},[550],[296,6317,6319,6323],{"className":6318},[572],[296,6320,5349],{"className":6321,"style":6322},[572,1957,1958],"margin-right:0.4445em;position:relative;top:-0.0011em;",[296,6324,6326],{"className":6325},[355],[296,6327,6329,6354],{"className":6328},[359,360],[296,6330,6332,6351],{"className":6331},[364],[296,6333,6336],{"className":6334,"style":6335},[368],"height:-0.6106em;",[296,6337,6339,6342],{"style":6338},"top:-1.7881em;margin-left:-0.4445em;margin-right:0.05em;",[296,6340],{"className":6341,"style":377},[376],[296,6343,6345],{"className":6344},[381,382,383,384],[296,6346,6348],{"className":6347},[346,384],[296,6349,454],{"className":6350,"style":600},[346,350,384],[296,6352,396],{"className":6353},[395],[296,6355,6357],{"className":6356},[364],[296,6358,6361],{"className":6359,"style":6360},[368],"height:0.9119em;",[296,6362],{},[296,6364],{"className":6365,"style":973},[550],[296,6367,1671],{"className":6368,"style":1632},[346,350],[296,6370,426],{"className":6371},[499],[296,6373,454],{"className":6374,"style":600},[346,350],[296,6376,820],{"className":6377},[346],[296,6379,323],{"className":6380,"style":391},[346,350],[296,6382,435],{"className":6383},[546],[296,6385,469],{"className":6386,"style":683},[346,350],[296,6388,426],{"className":6389},[499],[296,6391,454],{"className":6392,"style":600},[346,350],[296,6394,435],{"className":6395},[546],[296,6397,6398,6401],{"style":5916},[296,6399],{"className":6400,"style":4069},[376],[296,6402,6404,6407,6410,6413,6416,6459,6462,6505,6508,6511,6514,6517,6520,6523,6526,6529,6532],{"className":6403},[346],[296,6405],{"className":6406},[346],[296,6408],{"className":6409,"style":551},[550],[296,6411,438],{"className":6412},[555],[296,6414],{"className":6415,"style":551},[550],[296,6417,6419,6422],{"className":6418},[572],[296,6420,5349],{"className":6421,"style":6322},[572,1957,1958],[296,6423,6425],{"className":6424},[355],[296,6426,6428,6451],{"className":6427},[359,360],[296,6429,6431,6448],{"className":6430},[364],[296,6432,6434],{"className":6433,"style":6335},[368],[296,6435,6436,6439],{"style":6338},[296,6437],{"className":6438,"style":377},[376],[296,6440,6442],{"className":6441},[381,382,383,384],[296,6443,6445],{"className":6444},[346,384],[296,6446,454],{"className":6447,"style":600},[346,350,384],[296,6449,396],{"className":6450},[395],[296,6452,6454],{"className":6453},[364],[296,6455,6457],{"className":6456,"style":6360},[368],[296,6458],{},[296,6460],{"className":6461,"style":973},[550],[296,6463,6465,6468],{"className":6464},[346],[296,6466,803],{"className":6467},[346],[296,6469,6471],{"className":6470},[355],[296,6472,6474,6497],{"className":6473},[359,360],[296,6475,6477,6494],{"className":6476},[364],[296,6478,6480],{"className":6479,"style":369},[368],[296,6481,6482,6485],{"style":1005},[296,6483],{"className":6484,"style":377},[376],[296,6486,6488],{"className":6487},[381,382,383,384],[296,6489,6491],{"className":6490},[346,384],[296,6492,323],{"className":6493,"style":391},[346,350,384],[296,6495,396],{"className":6496},[395],[296,6498,6500],{"className":6499},[364],[296,6501,6503],{"className":6502,"style":403},[368],[296,6504],{},[296,6506,1671],{"className":6507,"style":1632},[346,350],[296,6509,426],{"className":6510},[499],[296,6512,454],{"className":6513,"style":600},[346,350],[296,6515,820],{"className":6516},[346],[296,6518,323],{"className":6519,"style":391},[346,350],[296,6521,435],{"className":6522},[546],[296,6524,469],{"className":6525,"style":683},[346,350],[296,6527,426],{"className":6528},[499],[296,6530,454],{"className":6531,"style":600},[346,350],[296,6533,435],{"className":6534},[546],[296,6536,6537,6540],{"style":5925},[296,6538],{"className":6539,"style":4069},[376],[296,6541,6543,6546,6549,6552,6555,6598,6601,6604,6607,6610,6613,6616,6619,6662,6665,6670,6673,6676,6679,6682,6685,6688,6691,6694,6697,6700],{"className":6542},[346],[296,6544],{"className":6545},[346],[296,6547],{"className":6548,"style":551},[550],[296,6550,438],{"className":6551},[555],[296,6553],{"className":6554,"style":551},[550],[296,6556,6558,6561],{"className":6557},[572],[296,6559,5349],{"className":6560,"style":6322},[572,1957,1958],[296,6562,6564],{"className":6563},[355],[296,6565,6567,6590],{"className":6566},[359,360],[296,6568,6570,6587],{"className":6569},[364],[296,6571,6573],{"className":6572,"style":6335},[368],[296,6574,6575,6578],{"style":6338},[296,6576],{"className":6577,"style":377},[376],[296,6579,6581],{"className":6580},[381,382,383,384],[296,6582,6584],{"className":6583},[346,384],[296,6585,454],{"className":6586,"style":600},[346,350,384],[296,6588,396],{"className":6589},[395],[296,6591,6593],{"className":6592},[364],[296,6594,6596],{"className":6595,"style":6360},[368],[296,6597],{},[296,6599],{"className":6600,"style":973},[550],[296,6602,1671],{"className":6603,"style":1632},[346,350],[296,6605,426],{"className":6606},[499],[296,6608,454],{"className":6609,"style":600},[346,350],[296,6611,820],{"className":6612},[346],[296,6614,323],{"className":6615,"style":391},[346,350],[296,6617,435],{"className":6618},[546],[296,6620,6622,6625],{"className":6621},[346],[296,6623,803],{"className":6624},[346],[296,6626,6628],{"className":6627},[355],[296,6629,6631,6654],{"className":6630},[359,360],[296,6632,6634,6651],{"className":6633},[364],[296,6635,6637],{"className":6636,"style":369},[368],[296,6638,6639,6642],{"style":1005},[296,6640],{"className":6641,"style":377},[376],[296,6643,6645],{"className":6644},[381,382,383,384],[296,6646,6648],{"className":6647},[346,384],[296,6649,323],{"className":6650,"style":391},[346,350,384],[296,6652,396],{"className":6653},[395],[296,6655,6657],{"className":6656},[364],[296,6658,6660],{"className":6659,"style":403},[368],[296,6661],{},[296,6663],{"className":6664,"style":973},[550],[296,6666,2312,6668],{"className":6667},[572],[296,6669,2316],{"style":2315},[296,6671],{"className":6672,"style":973},[550],[296,6674,1671],{"className":6675,"style":1632},[346,350],[296,6677,426],{"className":6678},[499],[296,6680,454],{"className":6681,"style":600},[346,350],[296,6683,820],{"className":6684},[346],[296,6686,323],{"className":6687,"style":391},[346,350],[296,6689,435],{"className":6690},[546],[296,6692,469],{"className":6693,"style":683},[346,350],[296,6695,426],{"className":6696},[499],[296,6698,454],{"className":6699,"style":600},[346,350],[296,6701,435],{"className":6702},[546],[296,6704,6705,6708],{"style":5934},[296,6706],{"className":6707,"style":4069},[376],[296,6709,6711,6714,6717,6720,6723,6819,6822],{"className":6710},[346],[296,6712],{"className":6713},[346],[296,6715],{"className":6716,"style":551},[550],[296,6718,438],{"className":6719},[555],[296,6721],{"className":6722,"style":551},[550],[296,6724,6726],{"className":6725},[346],[296,6727,6729],{"className":6728},[572,573],[296,6730,6732,6811],{"className":6731},[359,360],[296,6733,6735,6808],{"className":6734},[364],[296,6736,6738,6798],{"className":6737,"style":583},[368],[296,6739,6740,6743],{"style":586},[296,6741],{"className":6742,"style":590},[376],[296,6744,6746],{"className":6745},[381,382,383,384],[296,6747,6749,6752,6755],{"className":6748},[346,384],[296,6750,454],{"className":6751,"style":600},[346,350,384],[296,6753,457],{"className":6754},[555,384],[296,6756,6758,6761],{"className":6757},[346,384],[296,6759,320],{"className":6760,"style":351},[346,350,384],[296,6762,6764],{"className":6763},[355],[296,6765,6767,6790],{"className":6766},[359,360],[296,6768,6770,6787],{"className":6769},[364],[296,6771,6773],{"className":6772,"style":622},[368],[296,6774,6775,6778],{"style":625},[296,6776],{"className":6777,"style":629},[376],[296,6779,6781],{"className":6780},[381,633,634,384],[296,6782,6784],{"className":6783},[346,384],[296,6785,323],{"className":6786,"style":391},[346,350,384],[296,6788,396],{"className":6789},[395],[296,6791,6793],{"className":6792},[364],[296,6794,6796],{"className":6795,"style":650},[368],[296,6797],{},[296,6799,6800,6803],{"style":655},[296,6801],{"className":6802,"style":590},[376],[296,6804,6805],{},[296,6806,449],{"className":6807},[572,664],[296,6809,396],{"className":6810},[395],[296,6812,6814],{"className":6813},[364],[296,6815,6817],{"className":6816,"style":674},[368],[296,6818],{},[296,6820],{"className":6821,"style":973},[550],[296,6823,6825,6828,6871,6874,6879,6882,6885,6888,6891,6894,6897,6900,6903,6906,6909,6912],{"className":6824},[977],[296,6826,466],{"className":6827,"style":1085},[499,1084],[296,6829,6831,6834],{"className":6830},[346],[296,6832,803],{"className":6833},[346],[296,6835,6837],{"className":6836},[355],[296,6838,6840,6863],{"className":6839},[359,360],[296,6841,6843,6860],{"className":6842},[364],[296,6844,6846],{"className":6845,"style":369},[368],[296,6847,6848,6851],{"style":1005},[296,6849],{"className":6850,"style":377},[376],[296,6852,6854],{"className":6853},[381,382,383,384],[296,6855,6857],{"className":6856},[346,384],[296,6858,323],{"className":6859,"style":391},[346,350,384],[296,6861,396],{"className":6862},[395],[296,6864,6866],{"className":6865},[364],[296,6867,6869],{"className":6868,"style":403},[368],[296,6870],{},[296,6872],{"className":6873,"style":973},[550],[296,6875,2312,6877],{"className":6876},[572],[296,6878,2316],{"style":2315},[296,6880],{"className":6881,"style":973},[550],[296,6883,1671],{"className":6884,"style":1632},[346,350],[296,6886,426],{"className":6887},[499],[296,6889,454],{"className":6890,"style":600},[346,350],[296,6892,820],{"className":6893},[346],[296,6895,323],{"className":6896,"style":391},[346,350],[296,6898,435],{"className":6899},[546],[296,6901,469],{"className":6902,"style":683},[346,350],[296,6904,426],{"className":6905},[499],[296,6907,454],{"className":6908,"style":600},[346,350],[296,6910,435],{"className":6911},[546],[296,6913,478],{"className":6914,"style":1085},[546,1084],[296,6916,6917,6920],{"style":5943},[296,6918],{"className":6919,"style":4069},[376],[296,6921,6923,6926,6929,6932,6935,7031,7034],{"className":6922},[346],[296,6924],{"className":6925},[346],[296,6927],{"className":6928,"style":551},[550],[296,6930,438],{"className":6931},[555],[296,6933],{"className":6934,"style":551},[550],[296,6936,6938],{"className":6937},[346],[296,6939,6941],{"className":6940},[572,573],[296,6942,6944,7023],{"className":6943},[359,360],[296,6945,6947,7020],{"className":6946},[364],[296,6948,6950,7010],{"className":6949,"style":583},[368],[296,6951,6952,6955],{"style":586},[296,6953],{"className":6954,"style":590},[376],[296,6956,6958],{"className":6957},[381,382,383,384],[296,6959,6961,6964,6967],{"className":6960},[346,384],[296,6962,454],{"className":6963,"style":600},[346,350,384],[296,6965,457],{"className":6966},[555,384],[296,6968,6970,6973],{"className":6969},[346,384],[296,6971,320],{"className":6972,"style":351},[346,350,384],[296,6974,6976],{"className":6975},[355],[296,6977,6979,7002],{"className":6978},[359,360],[296,6980,6982,6999],{"className":6981},[364],[296,6983,6985],{"className":6984,"style":622},[368],[296,6986,6987,6990],{"style":625},[296,6988],{"className":6989,"style":629},[376],[296,6991,6993],{"className":6992},[381,633,634,384],[296,6994,6996],{"className":6995},[346,384],[296,6997,323],{"className":6998,"style":391},[346,350,384],[296,7000,396],{"className":7001},[395],[296,7003,7005],{"className":7004},[364],[296,7006,7008],{"className":7007,"style":650},[368],[296,7009],{},[296,7011,7012,7015],{"style":655},[296,7013],{"className":7014,"style":590},[376],[296,7016,7017],{},[296,7018,449],{"className":7019},[572,664],[296,7021,396],{"className":7022},[395],[296,7024,7026],{"className":7025},[364],[296,7027,7029],{"className":7028,"style":674},[368],[296,7030],{},[296,7032],{"className":7033,"style":973},[550],[296,7035,7037,7044,7111,7114,7157,7160,7165,7168,7211,7214,7254,7257,7297,7300,7303,7306,7309,7312],{"className":7036},[977],[296,7038,7040],{"className":7039,"style":1085},[499,1084],[296,7041,466],{"className":7042},[3026,7043],"size4",[296,7045,7047],{"className":7046},[572,573],[296,7048,7050,7103],{"className":7049},[359,360],[296,7051,7053,7100],{"className":7052},[364],[296,7054,7056,7076,7086],{"className":7055,"style":1923},[368],[296,7057,7058,7061],{"style":1926},[296,7059],{"className":7060,"style":1930},[376],[296,7062,7064],{"className":7063},[381,382,383,384],[296,7065,7067,7070,7073],{"className":7066},[346,384],[296,7068,1711],{"className":7069},[346,350,384],[296,7071,438],{"className":7072},[555,384],[296,7074,1425],{"className":7075},[346,384],[296,7077,7078,7081],{"style":1948},[296,7079],{"className":7080,"style":1930},[376],[296,7082,7083],{},[296,7084,2704],{"className":7085},[572,1957,1958],[296,7087,7088,7091],{"style":1961},[296,7089],{"className":7090,"style":1930},[376],[296,7092,7094],{"className":7093},[381,382,383,384],[296,7095,7097],{"className":7096},[346,384],[296,7098,1454],{"className":7099,"style":1632},[346,350,384],[296,7101,396],{"className":7102},[395],[296,7104,7106],{"className":7105},[364],[296,7107,7109],{"className":7108,"style":1983},[368],[296,7110],{},[296,7112],{"className":7113,"style":973},[550],[296,7115,7117,7120],{"className":7116},[346],[296,7118,803],{"className":7119},[346],[296,7121,7123],{"className":7122},[355],[296,7124,7126,7149],{"className":7125},[359,360],[296,7127,7129,7146],{"className":7128},[364],[296,7130,7132],{"className":7131,"style":369},[368],[296,7133,7134,7137],{"style":1005},[296,7135],{"className":7136,"style":377},[376],[296,7138,7140],{"className":7139},[381,382,383,384],[296,7141,7143],{"className":7142},[346,384],[296,7144,323],{"className":7145,"style":391},[346,350,384],[296,7147,396],{"className":7148},[395],[296,7150,7152],{"className":7151},[364],[296,7153,7155],{"className":7154,"style":403},[368],[296,7156],{},[296,7158],{"className":7159,"style":973},[550],[296,7161,2312,7163],{"className":7162},[572],[296,7164,2316],{"style":2315},[296,7166],{"className":7167,"style":973},[550],[296,7169,7171,7174],{"className":7170},[346],[296,7172,320],{"className":7173,"style":351},[346,350],[296,7175,7177],{"className":7176},[355],[296,7178,7180,7203],{"className":7179},[359,360],[296,7181,7183,7200],{"className":7182},[364],[296,7184,7186],{"className":7185,"style":369},[368],[296,7187,7188,7191],{"style":372},[296,7189],{"className":7190,"style":377},[376],[296,7192,7194],{"className":7193},[381,382,383,384],[296,7195,7197],{"className":7196},[346,384],[296,7198,323],{"className":7199,"style":391},[346,350,384],[296,7201,396],{"className":7202},[395],[296,7204,7206],{"className":7205},[364],[296,7207,7209],{"className":7208,"style":403},[368],[296,7210],{},[296,7212,426],{"className":7213},[499],[296,7215,7217,7220],{"className":7216},[346],[296,7218,216],{"className":7219},[346,350],[296,7221,7223],{"className":7222},[355],[296,7224,7226,7246],{"className":7225},[359,360],[296,7227,7229,7243],{"className":7228},[364],[296,7230,7232],{"className":7231,"style":2065},[368],[296,7233,7234,7237],{"style":1005},[296,7235],{"className":7236,"style":377},[376],[296,7238,7240],{"className":7239},[381,382,383,384],[296,7241,1711],{"className":7242},[346,350,384],[296,7244,396],{"className":7245},[395],[296,7247,7249],{"className":7248},[364],[296,7250,7252],{"className":7251,"style":403},[368],[296,7253],{},[296,7255,820],{"className":7256},[346],[296,7258,7260,7263],{"className":7259},[346],[296,7261,1422],{"className":7262},[346,350],[296,7264,7266],{"className":7265},[355],[296,7267,7269,7289],{"className":7268},[359,360],[296,7270,7272,7286],{"className":7271},[364],[296,7273,7275],{"className":7274,"style":2065},[368],[296,7276,7277,7280],{"style":1005},[296,7278],{"className":7279,"style":377},[376],[296,7281,7283],{"className":7282},[381,382,383,384],[296,7284,1711],{"className":7285},[346,350,384],[296,7287,396],{"className":7288},[395],[296,7290,7292],{"className":7291},[364],[296,7293,7295],{"className":7294,"style":403},[368],[296,7296],{},[296,7298,435],{"className":7299},[546],[296,7301,469],{"className":7302,"style":683},[346,350],[296,7304,426],{"className":7305},[499],[296,7307,454],{"className":7308,"style":600},[346,350],[296,7310,435],{"className":7311},[546],[296,7313,7315],{"className":7314,"style":1085},[546,1084],[296,7316,478],{"className":7317},[3026,7043],[296,7319,396],{"className":7320},[395],[296,7322,7324],{"className":7323},[364],[296,7325,7327],{"className":7326,"style":6061},[368],[296,7328],{},[296,7330],{"className":7331,"style":7333},[7332],"arraycolsep","width:1em;",[296,7335,7337],{"className":7336},[4052],[296,7338,7340,7397],{"className":7339},[359,360],[296,7341,7343,7394],{"className":7342},[364],[296,7344,7346,7354,7362,7370,7378,7386],{"className":7345,"style":5800},[368],[296,7347,7348,7351],{"style":5803},[296,7349],{"className":7350,"style":4069},[376],[296,7352],{"className":7353},[346],[296,7355,7356,7359],{"style":5907},[296,7357],{"className":7358,"style":4069},[376],[296,7360],{"className":7361},[346],[296,7363,7364,7367],{"style":5916},[296,7365],{"className":7366,"style":4069},[376],[296,7368],{"className":7369},[346],[296,7371,7372,7375],{"style":5925},[296,7373],{"className":7374,"style":4069},[376],[296,7376],{"className":7377},[346],[296,7379,7380,7383],{"style":5934},[296,7381],{"className":7382,"style":4069},[376],[296,7384],{"className":7385},[346],[296,7387,7388,7391],{"style":5943},[296,7389],{"className":7390,"style":4069},[376],[296,7392],{"className":7393},[346],[296,7395,396],{"className":7396},[395],[296,7398,7400],{"className":7399},[364],[296,7401,7403],{"className":7402,"style":6061},[368],[296,7404],{},[296,7406,7408],{"className":7407},[4169],[296,7409,7411,7523],{"className":7410},[359,360],[296,7412,7414,7520],{"className":7413},[364],[296,7415,7417,7428,7447,7464,7481,7498],{"className":7416,"style":5800},[368],[296,7418,7419,7422],{"style":5803},[296,7420],{"className":7421,"style":4069},[376],[296,7423,7425],{"className":7424},[346],[296,7426],{"className":7427},[346],[296,7429,7430,7433],{"style":5907},[296,7431],{"className":7432,"style":4069},[376],[296,7434,7436,7439],{"className":7435},[346],[296,7437],{"className":7438},[346],[296,7440,7443],{"className":7441},[346,7442],"text",[296,7444,5389],{"className":7445},[346,7446],"cjk_fallback",[296,7448,7449,7452],{"style":5916},[296,7450],{"className":7451,"style":4069},[376],[296,7453,7455,7458],{"className":7454},[346],[296,7456],{"className":7457},[346],[296,7459,7461],{"className":7460},[346,7442],[296,7462,5456],{"className":7463},[346,7446],[296,7465,7466,7469],{"style":5925},[296,7467],{"className":7468,"style":4069},[376],[296,7470,7472,7475],{"className":7471},[346],[296,7473],{"className":7474},[346],[296,7476,7478],{"className":7477},[346,7442],[296,7479,5539],{"className":7480},[346,7446],[296,7482,7483,7486],{"style":5934},[296,7484],{"className":7485,"style":4069},[376],[296,7487,7489,7492],{"className":7488},[346],[296,7490],{"className":7491},[346],[296,7493,7495],{"className":7494},[346,7442],[296,7496,5630],{"className":7497},[346,7446],[296,7499,7500,7503],{"style":5943},[296,7501],{"className":7502,"style":4069},[376],[296,7504,7506,7509],{"className":7505},[346],[296,7507],{"className":7508},[346],[296,7510,7512,7516],{"className":7511},[346,7442],[296,7513,7515],{"className":7514},[346,7446],"代入轨迹的",[296,7517,7519],{"className":7518},[346]," grad-log-prob",[296,7521,396],{"className":7522},[395],[296,7524,7526],{"className":7525},[364],[296,7527,7529],{"className":7528,"style":6061},[368],[296,7530],{},[212,7532,7533,7534,7729,7730,7802],{},"这是一个期望，意味着可以用样本均值来估计它。收集一组轨迹 ",[296,7535,7537,7591],{"className":7536},[299],[296,7538,7540],{"className":7539},[303],[305,7541,7542],{"xmlns":307},[309,7543,7544,7588],{},[312,7545,7546,7550,7552,7555,7562],{},[318,7547,7549],{"mathvariant":7548},"script","D",[423,7551,438],{},[423,7553,7554],{"stretchy":425},"{",[315,7556,7557,7559],{},[318,7558,454],{},[318,7560,7561],{},"i",[315,7563,7564,7567],{},[423,7565,7566],{"stretchy":425},"}",[312,7568,7569,7571,7573,7575,7577,7579,7581,7583,7585],{},[318,7570,7561],{},[423,7572,438],{},[778,7574,780],{},[423,7576,1428],{"separator":333},[318,7578,1439],{"mathvariant":802},[318,7580,1439],{"mathvariant":802},[318,7582,1439],{"mathvariant":802},[423,7584,1428],{"separator":333},[318,7586,7587],{},"N",[325,7589,7590],{"encoding":327},"\\mathcal{D} = \\{\\tau_i\\}_{i=1,...,N}",[296,7592,7594,7614],{"className":7593,"ariaHidden":333},[332],[296,7595,7597,7601,7605,7608,7611],{"className":7596},[337],[296,7598],{"className":7599,"style":7600},[341],"height:0.6833em;",[296,7602,7549],{"className":7603,"style":391},[346,7604],"mathcal",[296,7606],{"className":7607,"style":551},[550],[296,7609,438],{"className":7610},[555],[296,7612],{"className":7613,"style":551},[550],[296,7615,7617,7621,7624,7666],{"className":7616},[337],[296,7618],{"className":7619,"style":7620},[341],"height:1.0361em;vertical-align:-0.2861em;",[296,7622,7554],{"className":7623},[499],[296,7625,7627,7630],{"className":7626},[346],[296,7628,454],{"className":7629,"style":600},[346,350],[296,7631,7633],{"className":7632},[355],[296,7634,7636,7658],{"className":7635},[359,360],[296,7637,7639,7655],{"className":7638},[364],[296,7640,7643],{"className":7641,"style":7642},[368],"height:0.3117em;",[296,7644,7646,7649],{"style":7645},"top:-2.55em;margin-left:-0.1132em;margin-right:0.05em;",[296,7647],{"className":7648,"style":377},[376],[296,7650,7652],{"className":7651},[381,382,383,384],[296,7653,7561],{"className":7654},[346,350,384],[296,7656,396],{"className":7657},[395],[296,7659,7661],{"className":7660},[364],[296,7662,7664],{"className":7663,"style":403},[368],[296,7665],{},[296,7667,7669,7672],{"className":7668},[546],[296,7670,7566],{"className":7671},[546],[296,7673,7675],{"className":7674},[355],[296,7676,7678,7720],{"className":7677},[359,360],[296,7679,7681,7717],{"className":7680},[364],[296,7682,7684],{"className":7683,"style":1617},[368],[296,7685,7686,7689],{"style":1005},[296,7687],{"className":7688,"style":377},[376],[296,7690,7692],{"className":7691},[381,382,383,384],[296,7693,7695,7698,7701,7704,7707,7710,7713],{"className":7694},[346,384],[296,7696,7561],{"className":7697},[346,350,384],[296,7699,438],{"className":7700},[555,384],[296,7702,780],{"className":7703},[346,384],[296,7705,1428],{"className":7706},[1539,384],[296,7708,1592],{"className":7709},[346,384],[296,7711,1428],{"className":7712},[1539,384],[296,7714,7587],{"className":7715,"style":7716},[346,350,384],"margin-right:0.109em;",[296,7718,396],{"className":7719},[395],[296,7721,7723],{"className":7722},[364],[296,7724,7727],{"className":7725,"style":7726},[368],"height:0.2861em;",[296,7728],{},"，每条轨迹都让智能体按策略 ",[296,7731,7733,7750],{"className":7732},[299],[296,7734,7736],{"className":7735},[303],[305,7737,7738],{"xmlns":307},[309,7739,7740,7748],{},[312,7741,7742],{},[315,7743,7744,7746],{},[318,7745,320],{},[318,7747,323],{},[325,7749,328],{"encoding":327},[296,7751,7753],{"className":7752,"ariaHidden":333},[332],[296,7754,7756,7759],{"className":7755},[337],[296,7757],{"className":7758,"style":342},[341],[296,7760,7762,7765],{"className":7761},[346],[296,7763,320],{"className":7764,"style":351},[346,350],[296,7766,7768],{"className":7767},[355],[296,7769,7771,7794],{"className":7770},[359,360],[296,7772,7774,7791],{"className":7773},[364],[296,7775,7777],{"className":7776,"style":369},[368],[296,7778,7779,7782],{"style":372},[296,7780],{"className":7781,"style":377},[376],[296,7783,7785],{"className":7784},[381,382,383,384],[296,7786,7788],{"className":7787},[346,384],[296,7789,323],{"className":7790,"style":391},[346,350,384],[296,7792,396],{"className":7793},[395],[296,7795,7797],{"className":7796},[364],[296,7798,7800],{"className":7799,"style":403},[368],[296,7801],{}," 在环境中行动得到，策略梯度就可以这样估计：",[296,7804,7806],{"className":7805},[751],[296,7807,7809,7913],{"className":7808},[299],[296,7810,7812],{"className":7811},[303],[305,7813,7814],{"xmlns":307,"display":760},[309,7815,7816,7910],{},[312,7817,7818,7826,7828,7841,7854,7868,7874,7876,7878,7884,7886,7892,7894,7900,7902,7904,7906,7908],{},[7819,7820,7821,7823],"mover",{"accent":333},[318,7822,2316],{},[423,7824,7825],{},"^",[423,7827,438],{},[7829,7830,7831,7833],"mfrac",{},[778,7832,780],{},[312,7834,7835,7837,7839],{},[318,7836,820],{"mathvariant":802},[318,7838,7549],{"mathvariant":7548},[318,7840,820],{"mathvariant":802},[442,7842,7843,7845],{},[423,7844,2704],{},[312,7846,7847,7849,7852],{},[318,7848,454],{},[423,7850,7851],{},"∈",[318,7853,7549],{"mathvariant":7548},[1702,7855,7856,7858,7866],{},[423,7857,2704],{},[312,7859,7860,7862,7864],{},[318,7861,1711],{},[423,7863,438],{},[778,7865,1425],{},[318,7867,1454],{},[315,7869,7870,7872],{},[318,7871,803],{"mathvariant":802},[318,7873,323],{},[318,7875,2289],{},[423,7877,2292],{},[315,7879,7880,7882],{},[318,7881,320],{},[318,7883,323],{},[423,7885,426],{"stretchy":425},[315,7887,7888,7890],{},[318,7889,216],{},[318,7891,1711],{},[318,7893,820],{"mathvariant":802},[315,7895,7896,7898],{},[318,7897,1422],{},[318,7899,1711],{},[423,7901,435],{"stretchy":425},[318,7903,469],{},[423,7905,426],{"stretchy":425},[318,7907,454],{},[423,7909,435],{"stretchy":425},[325,7911,7912],{"encoding":327},"\\hat{g} = \\frac{1}{|\\mathcal{D}|} \\sum_{\\tau \\in \\mathcal{D}} \\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) R(\\tau)",[296,7914,7916,7977],{"className":7915,"ariaHidden":333},[332],[296,7917,7919,7922,7968,7971,7974],{"className":7918},[337],[296,7920],{"className":7921,"style":2308},[341],[296,7923,7926],{"className":7924},[346,7925],"accent",[296,7927,7929,7959],{"className":7928},[359,360],[296,7930,7932,7956],{"className":7931},[364],[296,7933,7935,7943],{"className":7934,"style":3379},[368],[296,7936,7937,7940],{"style":655},[296,7938],{"className":7939,"style":590},[376],[296,7941,2316],{"className":7942,"style":351},[346,350],[296,7944,7945,7948],{"style":655},[296,7946],{"className":7947,"style":590},[376],[296,7949,7953],{"className":7950,"style":7952},[7951],"accent-body","left:-0.2222em;",[296,7954,7825],{"className":7955},[346],[296,7957,396],{"className":7958},[395],[296,7960,7962],{"className":7961},[364],[296,7963,7966],{"className":7964,"style":7965},[368],"height:0.1944em;",[296,7967],{},[296,7969],{"className":7970,"style":551},[550],[296,7972,438],{"className":7973},[555],[296,7975],{"className":7976,"style":551},[550],[296,7978,7980,7984,8059,8062,8118,8121,8188,8191,8234,8237,8242,8245,8288,8291,8331,8334,8374,8377,8380,8383,8386],{"className":7979},[337],[296,7981],{"className":7982,"style":7983},[341],"height:3.15em;vertical-align:-1.3217em;",[296,7985,7987,7990,8056],{"className":7986},[346],[296,7988],{"className":7989},[499,984],[296,7991,7993],{"className":7992},[7829],[296,7994,7996,8047],{"className":7995},[359,360],[296,7997,7999,8044],{"className":7998},[364],[296,8000,8003,8021,8032],{"className":8001,"style":8002},[368],"height:1.3214em;",[296,8004,8006,8009],{"style":8005},"top:-2.314em;",[296,8007],{"className":8008,"style":590},[376],[296,8010,8012,8015,8018],{"className":8011},[346],[296,8013,820],{"className":8014},[346],[296,8016,7549],{"className":8017,"style":391},[346,7604],[296,8019,820],{"className":8020},[346],[296,8022,8024,8027],{"style":8023},"top:-3.23em;",[296,8025],{"className":8026,"style":590},[376],[296,8028],{"className":8029,"style":8031},[8030],"frac-line","border-bottom-width:0.04em;",[296,8033,8035,8038],{"style":8034},"top:-3.677em;",[296,8036],{"className":8037,"style":590},[376],[296,8039,8041],{"className":8040},[346],[296,8042,780],{"className":8043},[346],[296,8045,396],{"className":8046},[395],[296,8048,8050],{"className":8049},[364],[296,8051,8054],{"className":8052,"style":8053},[368],"height:0.936em;",[296,8055],{},[296,8057],{"className":8058},[546,984],[296,8060],{"className":8061,"style":973},[550],[296,8063,8065],{"className":8064},[572,573],[296,8066,8068,8109],{"className":8067},[359,360],[296,8069,8071,8106],{"className":8070},[364],[296,8072,8075,8096],{"className":8073,"style":8074},[368],"height:1.05em;",[296,8076,8078,8081],{"style":8077},"top:-1.8557em;margin-left:0em;",[296,8079],{"className":8080,"style":1930},[376],[296,8082,8084],{"className":8083},[381,382,383,384],[296,8085,8087,8090,8093],{"className":8086},[346,384],[296,8088,454],{"className":8089,"style":600},[346,350,384],[296,8091,7851],{"className":8092},[555,384],[296,8094,7549],{"className":8095,"style":391},[346,7604,384],[296,8097,8098,8101],{"style":1948},[296,8099],{"className":8100,"style":1930},[376],[296,8102,8103],{},[296,8104,2704],{"className":8105},[572,1957,1958],[296,8107,396],{"className":8108},[395],[296,8110,8112],{"className":8111},[364],[296,8113,8116],{"className":8114,"style":8115},[368],"height:1.3217em;",[296,8117],{},[296,8119],{"className":8120,"style":973},[550],[296,8122,8124],{"className":8123},[572,573],[296,8125,8127,8180],{"className":8126},[359,360],[296,8128,8130,8177],{"className":8129},[364],[296,8131,8133,8153,8163],{"className":8132,"style":1923},[368],[296,8134,8135,8138],{"style":1926},[296,8136],{"className":8137,"style":1930},[376],[296,8139,8141],{"className":8140},[381,382,383,384],[296,8142,8144,8147,8150],{"className":8143},[346,384],[296,8145,1711],{"className":8146},[346,350,384],[296,8148,438],{"className":8149},[555,384],[296,8151,1425],{"className":8152},[346,384],[296,8154,8155,8158],{"style":1948},[296,8156],{"className":8157,"style":1930},[376],[296,8159,8160],{},[296,8161,2704],{"className":8162},[572,1957,1958],[296,8164,8165,8168],{"style":1961},[296,8166],{"className":8167,"style":1930},[376],[296,8169,8171],{"className":8170},[381,382,383,384],[296,8172,8174],{"className":8173},[346,384],[296,8175,1454],{"className":8176,"style":1632},[346,350,384],[296,8178,396],{"className":8179},[395],[296,8181,8183],{"className":8182},[364],[296,8184,8186],{"className":8185,"style":1983},[368],[296,8187],{},[296,8189],{"className":8190,"style":973},[550],[296,8192,8194,8197],{"className":8193},[346],[296,8195,803],{"className":8196},[346],[296,8198,8200],{"className":8199},[355],[296,8201,8203,8226],{"className":8202},[359,360],[296,8204,8206,8223],{"className":8205},[364],[296,8207,8209],{"className":8208,"style":369},[368],[296,8210,8211,8214],{"style":1005},[296,8212],{"className":8213,"style":377},[376],[296,8215,8217],{"className":8216},[381,382,383,384],[296,8218,8220],{"className":8219},[346,384],[296,8221,323],{"className":8222,"style":391},[346,350,384],[296,8224,396],{"className":8225},[395],[296,8227,8229],{"className":8228},[364],[296,8230,8232],{"className":8231,"style":403},[368],[296,8233],{},[296,8235],{"className":8236,"style":973},[550],[296,8238,2312,8240],{"className":8239},[572],[296,8241,2316],{"style":2315},[296,8243],{"className":8244,"style":973},[550],[296,8246,8248,8251],{"className":8247},[346],[296,8249,320],{"className":8250,"style":351},[346,350],[296,8252,8254],{"className":8253},[355],[296,8255,8257,8280],{"className":8256},[359,360],[296,8258,8260,8277],{"className":8259},[364],[296,8261,8263],{"className":8262,"style":369},[368],[296,8264,8265,8268],{"style":372},[296,8266],{"className":8267,"style":377},[376],[296,8269,8271],{"className":8270},[381,382,383,384],[296,8272,8274],{"className":8273},[346,384],[296,8275,323],{"className":8276,"style":391},[346,350,384],[296,8278,396],{"className":8279},[395],[296,8281,8283],{"className":8282},[364],[296,8284,8286],{"className":8285,"style":403},[368],[296,8287],{},[296,8289,426],{"className":8290},[499],[296,8292,8294,8297],{"className":8293},[346],[296,8295,216],{"className":8296},[346,350],[296,8298,8300],{"className":8299},[355],[296,8301,8303,8323],{"className":8302},[359,360],[296,8304,8306,8320],{"className":8305},[364],[296,8307,8309],{"className":8308,"style":2065},[368],[296,8310,8311,8314],{"style":1005},[296,8312],{"className":8313,"style":377},[376],[296,8315,8317],{"className":8316},[381,382,383,384],[296,8318,1711],{"className":8319},[346,350,384],[296,8321,396],{"className":8322},[395],[296,8324,8326],{"className":8325},[364],[296,8327,8329],{"className":8328,"style":403},[368],[296,8330],{},[296,8332,820],{"className":8333},[346],[296,8335,8337,8340],{"className":8336},[346],[296,8338,1422],{"className":8339},[346,350],[296,8341,8343],{"className":8342},[355],[296,8344,8346,8366],{"className":8345},[359,360],[296,8347,8349,8363],{"className":8348},[364],[296,8350,8352],{"className":8351,"style":2065},[368],[296,8353,8354,8357],{"style":1005},[296,8355],{"className":8356,"style":377},[376],[296,8358,8360],{"className":8359},[381,382,383,384],[296,8361,1711],{"className":8362},[346,350,384],[296,8364,396],{"className":8365},[395],[296,8367,8369],{"className":8368},[364],[296,8370,8372],{"className":8371,"style":403},[368],[296,8373],{},[296,8375,435],{"className":8376},[546],[296,8378,469],{"className":8379,"style":683},[346,350],[296,8381,426],{"className":8382},[499],[296,8384,454],{"className":8385,"style":600},[346,350],[296,8387,435],{"className":8388},[546],[212,8390,8391,8392,8431,8432,8461,8462,8490],{},"其中 ",[296,8393,8395,8413],{"className":8394},[299],[296,8396,8398],{"className":8397},[303],[305,8399,8400],{"xmlns":307},[309,8401,8402,8410],{},[312,8403,8404,8406,8408],{},[318,8405,820],{"mathvariant":802},[318,8407,7549],{"mathvariant":7548},[318,8409,820],{"mathvariant":802},[325,8411,8412],{"encoding":327},"|\\mathcal{D}|",[296,8414,8416],{"className":8415,"ariaHidden":333},[332],[296,8417,8419,8422,8425,8428],{"className":8418},[337],[296,8420],{"className":8421,"style":491},[341],[296,8423,820],{"className":8424},[346],[296,8426,7549],{"className":8427,"style":391},[346,7604],[296,8429,820],{"className":8430},[346]," 是 ",[296,8433,8435,8449],{"className":8434},[299],[296,8436,8438],{"className":8437},[303],[305,8439,8440],{"xmlns":307},[309,8441,8442,8446],{},[312,8443,8444],{},[318,8445,7549],{"mathvariant":7548},[325,8447,8448],{"encoding":327},"\\mathcal{D}",[296,8450,8452],{"className":8451,"ariaHidden":333},[332],[296,8453,8455,8458],{"className":8454},[337],[296,8456],{"className":8457,"style":7600},[341],[296,8459,7549],{"className":8460,"style":391},[346,7604]," 里的轨迹条数（这里是 ",[296,8463,8465,8478],{"className":8464},[299],[296,8466,8468],{"className":8467},[303],[305,8469,8470],{"xmlns":307},[309,8471,8472,8476],{},[312,8473,8474],{},[318,8475,7587],{},[325,8477,7587],{"encoding":327},[296,8479,8481],{"className":8480,"ariaHidden":333},[332],[296,8482,8484,8487],{"className":8483},[337],[296,8485],{"className":8486,"style":7600},[341],[296,8488,7587],{"className":8489,"style":7716},[346,350],"）。",[212,8492,8493,8494,8656],{},"最后这个式子就是我们想要的「可计算表达式」的最简单的版本。只要策略的表示方式让我们能算出 ",[296,8495,8497,8535],{"className":8496},[299],[296,8498,8500],{"className":8499},[303],[305,8501,8502],{"xmlns":307},[309,8503,8504,8532],{},[312,8505,8506,8512,8514,8516,8522,8524,8526,8528,8530],{},[315,8507,8508,8510],{},[318,8509,803],{"mathvariant":802},[318,8511,323],{},[318,8513,2289],{},[423,8515,2292],{},[315,8517,8518,8520],{},[318,8519,320],{},[318,8521,323],{},[423,8523,426],{"stretchy":425},[318,8525,216],{},[318,8527,820],{"mathvariant":802},[318,8529,1422],{},[423,8531,435],{"stretchy":425},[325,8533,8534],{"encoding":327},"\\nabla_{\\theta} \\log \\pi_{\\theta}(a|s)",[296,8536,8538],{"className":8537,"ariaHidden":333},[332],[296,8539,8541,8544,8587,8590,8595,8598,8641,8644,8647,8650,8653],{"className":8540},[337],[296,8542],{"className":8543,"style":491},[341],[296,8545,8547,8550],{"className":8546},[346],[296,8548,803],{"className":8549},[346],[296,8551,8553],{"className":8552},[355],[296,8554,8556,8579],{"className":8555},[359,360],[296,8557,8559,8576],{"className":8558},[364],[296,8560,8562],{"className":8561,"style":369},[368],[296,8563,8564,8567],{"style":1005},[296,8565],{"className":8566,"style":377},[376],[296,8568,8570],{"className":8569},[381,382,383,384],[296,8571,8573],{"className":8572},[346,384],[296,8574,323],{"className":8575,"style":391},[346,350,384],[296,8577,396],{"className":8578},[395],[296,8580,8582],{"className":8581},[364],[296,8583,8585],{"className":8584,"style":403},[368],[296,8586],{},[296,8588],{"className":8589,"style":973},[550],[296,8591,2312,8593],{"className":8592},[572],[296,8594,2316],{"style":2315},[296,8596],{"className":8597,"style":973},[550],[296,8599,8601,8604],{"className":8600},[346],[296,8602,320],{"className":8603,"style":351},[346,350],[296,8605,8607],{"className":8606},[355],[296,8608,8610,8633],{"className":8609},[359,360],[296,8611,8613,8630],{"className":8612},[364],[296,8614,8616],{"className":8615,"style":369},[368],[296,8617,8618,8621],{"style":372},[296,8619],{"className":8620,"style":377},[376],[296,8622,8624],{"className":8623},[381,382,383,384],[296,8625,8627],{"className":8626},[346,384],[296,8628,323],{"className":8629,"style":391},[346,350,384],[296,8631,396],{"className":8632},[395],[296,8634,8636],{"className":8635},[364],[296,8637,8639],{"className":8638,"style":403},[368],[296,8640],{},[296,8642,426],{"className":8643},[499],[296,8645,216],{"className":8646},[346,350],[296,8648,820],{"className":8649},[346],[296,8651,1422],{"className":8652},[346,350],[296,8654,435],{"className":8655},[546],"，并且能把策略放进环境里跑、收集轨迹数据集，我们就能算出策略梯度、走一步更新。",[289,8658,8659],{"id":8659},"实现最简单的策略梯度",[212,8661,8662,8663,8667,8668,8673],{},"这个最简单版本策略梯度算法的 PyTorch 短实现放在 ",[8664,8665,8666],"code",{},"spinup\u002Fexamples\u002Fpytorch\u002Fpg_math\u002F1_simple_pg.py","（也可以",[216,8669,8672],{"href":8670,"rel":8671},"https:\u002F\u002Fgithub.com\u002Fopenai\u002Fspinningup\u002Fblob\u002Fmaster\u002Fspinup\u002Fexamples\u002Fpytorch\u002Fpg_math\u002F1_simple_pg.py",[220],"在 GitHub 上查看","）。它只有 128 行，我们强烈建议通读一遍。这里不逐行讲整个文件，只挑几块重要的部分解释。",[208,8675,8677],{"title":8676},"你应该知道",[212,8678,8679,8680,8685],{},"本节最初写的是 Tensorflow 例子，旧版 Tensorflow 的内容见",[216,8681,8684],{"href":8682,"rel":8683},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Fextra_tf_pg_implementation.html#implementing-the-simplest-policy-gradient",[220],"这里","。",[212,8687,8688],{},[247,8689,8690],{},"1. 搭建策略网络。",[8692,8693,8697],"pre",{"className":8694,"code":8695,"language":8696,"meta":40,"style":40},"language-python3 shiki shiki-themes github-light github-dark","# 搭建策略网络的核心\nlogits_net = mlp(sizes=[obs_dim]+hidden_sizes+[n_acts])\n\n# 构造计算动作分布的函数\ndef get_policy(obs):\n    logits = logits_net(obs)\n    return Categorical(logits=logits)\n\n# 构造选动作的函数（输出整数动作，从策略中采样）\ndef get_action(obs):\n    return get_policy(obs).sample().item()\n","python3",[8664,8698,8699,8705,8711,8717,8723,8729,8735,8741,8746,8752,8758],{"__ignoreMap":40},[296,8700,8702],{"class":4380,"line":8701},1,[296,8703,8704],{},"# 搭建策略网络的核心\n",[296,8706,8708],{"class":4380,"line":8707},2,[296,8709,8710],{},"logits_net = mlp(sizes=[obs_dim]+hidden_sizes+[n_acts])\n",[296,8712,8714],{"class":4380,"line":8713},3,[296,8715,8716],{"emptyLinePlaceholder":36},"\n",[296,8718,8720],{"class":4380,"line":8719},4,[296,8721,8722],{},"# 构造计算动作分布的函数\n",[296,8724,8726],{"class":4380,"line":8725},5,[296,8727,8728],{},"def get_policy(obs):\n",[296,8730,8732],{"class":4380,"line":8731},6,[296,8733,8734],{},"    logits = logits_net(obs)\n",[296,8736,8738],{"class":4380,"line":8737},7,[296,8739,8740],{},"    return Categorical(logits=logits)\n",[296,8742,8744],{"class":4380,"line":8743},8,[296,8745,8716],{"emptyLinePlaceholder":36},[296,8747,8749],{"class":4380,"line":8748},9,[296,8750,8751],{},"# 构造选动作的函数（输出整数动作，从策略中采样）\n",[296,8753,8755],{"class":4380,"line":8754},10,[296,8756,8757],{},"def get_action(obs):\n",[296,8759,8761],{"class":4380,"line":8760},11,[296,8762,8763],{},"    return get_policy(obs).sample().item()\n",[212,8765,8766,8767,8490,8771,8774,8775,8778,8779,8781,8782,8785,8786,8789,8790,8795],{},"这一块搭建了前馈神经网络分类策略所需的模块和函数（想复习可以看",[216,8768,8770],{"href":8769},"\u002Fposts\u002Fspinning-up-rl-intro#%E9%9A%8F%E6%9C%BA%E5%9E%8B%E7%AD%96%E7%95%A5","第一篇的「随机型策略」一节",[8664,8772,8773],{},"logits_net"," 模块的输出可以用来构造动作的对数概率和概率，",[8664,8776,8777],{},"get_action"," 函数根据 logits 算出的概率采样动作。（注意：这个 ",[8664,8780,8777],{}," 假定每次只传入一个 ",[8664,8783,8784],{},"obs","，因此也只输出一个整数动作。这就是它用 ",[8664,8787,8788],{},".item()"," 的原因，这个方法用来",[216,8791,8794],{"href":8792,"rel":8793},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Ftensors.html#torch.Tensor.item",[220],"取出只含单个元素的 Tensor 的内容","。）",[212,8797,8798,8799,8802,8803,8806,8807,8812],{},"这个例子里有大量工作是由源文件第 36 行的 ",[8664,8800,8801],{},"Categorical"," 对象完成的。它是 PyTorch 的 ",[8664,8804,8805],{},"Distribution"," 对象，封装了与概率分布相关的一些数学函数：既有从分布采样的方法（第 40 行用到了），也有计算给定样本对数概率的方法（后面会用到）。PyTorch 的 distributions 对 RL 非常有用，建议翻一翻",[216,8808,8811],{"href":8809,"rel":8810},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Fdistributions.html",[220],"它们的文档","，感受一下怎么用。",[208,8814,8815,8922],{"title":8676},[212,8816,8817,8818,8891,8892,8921],{},"友好提醒！说一个分类分布有「logits」，意思是各结果的概率由 logits 的 softmax 函数给出。也就是说，logits 为 ",[296,8819,8821,8840],{"className":8820},[299],[296,8822,8824],{"className":8823},[303],[305,8825,8826],{"xmlns":307},[309,8827,8828,8837],{},[312,8829,8830],{},[315,8831,8832,8834],{},[318,8833,2295],{},[318,8835,8836],{},"j",[325,8838,8839],{"encoding":327},"x_j",[296,8841,8843],{"className":8842,"ariaHidden":333},[332],[296,8844,8846,8850],{"className":8845},[337],[296,8847],{"className":8848,"style":8849},[341],"height:0.7167em;vertical-align:-0.2861em;",[296,8851,8853,8856],{"className":8852},[346],[296,8854,2295],{"className":8855},[346,350],[296,8857,8859],{"className":8858},[355],[296,8860,8862,8883],{"className":8861},[359,360],[296,8863,8865,8880],{"className":8864},[364],[296,8866,8868],{"className":8867,"style":7642},[368],[296,8869,8870,8873],{"style":1005},[296,8871],{"className":8872,"style":377},[376],[296,8874,8876],{"className":8875},[381,382,383,384],[296,8877,8836],{"className":8878,"style":8879},[346,350,384],"margin-right:0.0572em;",[296,8881,396],{"className":8882},[395],[296,8884,8886],{"className":8885},[364],[296,8887,8889],{"className":8888,"style":7726},[368],[296,8890],{}," 的分类分布下，动作 ",[296,8893,8895,8908],{"className":8894},[299],[296,8896,8898],{"className":8897},[303],[305,8899,8900],{"xmlns":307},[309,8901,8902,8906],{},[312,8903,8904],{},[318,8905,8836],{},[325,8907,8836],{"encoding":327},[296,8909,8911],{"className":8910,"ariaHidden":333},[332],[296,8912,8914,8918],{"className":8913},[337],[296,8915],{"className":8916,"style":8917},[341],"height:0.854em;vertical-align:-0.1944em;",[296,8919,8836],{"className":8920,"style":8879},[346,350]," 的概率是：",[296,8923,8925],{"className":8924},[751],[296,8926,8928,8989],{"className":8927},[299],[296,8929,8931],{"className":8930},[303],[305,8932,8933],{"xmlns":307,"display":760},[309,8934,8935,8986],{},[312,8936,8937,8943,8945],{},[315,8938,8939,8941],{},[318,8940,212],{},[318,8942,8836],{},[423,8944,438],{},[7829,8946,8947,8964],{},[312,8948,8949,8952,8954,8956,8962],{},[318,8950,8951],{},"exp",[423,8953,2292],{},[423,8955,426],{"stretchy":425},[315,8957,8958,8960],{},[318,8959,2295],{},[318,8961,8836],{},[423,8963,435],{"stretchy":425},[312,8965,8966,8972,8974,8976,8978,8984],{},[442,8967,8968,8970],{},[423,8969,2704],{},[318,8971,7561],{},[318,8973,8951],{},[423,8975,2292],{},[423,8977,426],{"stretchy":425},[315,8979,8980,8982],{},[318,8981,2295],{},[318,8983,7561],{},[423,8985,435],{"stretchy":425},[325,8987,8988],{"encoding":327},"p_j = \\frac{\\exp(x_j)}{\\sum_{i} \\exp(x_i)}",[296,8990,8992,9047],{"className":8991,"ariaHidden":333},[332],[296,8993,8995,8998,9038,9041,9044],{"className":8994},[337],[296,8996],{"className":8997,"style":8849},[341],[296,8999,9001,9004],{"className":9000},[346],[296,9002,212],{"className":9003},[346,350],[296,9005,9007],{"className":9006},[355],[296,9008,9010,9030],{"className":9009},[359,360],[296,9011,9013,9027],{"className":9012},[364],[296,9014,9016],{"className":9015,"style":7642},[368],[296,9017,9018,9021],{"style":1005},[296,9019],{"className":9020,"style":377},[376],[296,9022,9024],{"className":9023},[381,382,383,384],[296,9025,8836],{"className":9026,"style":8879},[346,350,384],[296,9028,396],{"className":9029},[395],[296,9031,9033],{"className":9032},[364],[296,9034,9036],{"className":9035,"style":7726},[368],[296,9037],{},[296,9039],{"className":9040,"style":551},[550],[296,9042,438],{"className":9043},[555],[296,9045],{"className":9046,"style":551},[550],[296,9048,9050,9054],{"className":9049},[337],[296,9051],{"className":9052,"style":9053},[341],"height:2.4127em;vertical-align:-0.9857em;",[296,9055,9057,9060,9258],{"className":9056},[346],[296,9058],{"className":9059},[499,984],[296,9061,9063],{"className":9062},[7829],[296,9064,9066,9249],{"className":9065},[359,360],[296,9067,9069,9246],{"className":9068},[364],[296,9070,9073,9181,9189],{"className":9071,"style":9072},[368],"height:1.427em;",[296,9074,9075,9078],{"style":8005},[296,9076],{"className":9077,"style":590},[376],[296,9079,9081,9129,9132,9135,9138,9178],{"className":9080},[346],[296,9082,9084,9089],{"className":9083},[572],[296,9085,2704],{"className":9086,"style":9088},[572,1957,9087],"small-op","position:relative;top:0em;",[296,9090,9092],{"className":9091},[355],[296,9093,9095,9120],{"className":9094},[359,360],[296,9096,9098,9117],{"className":9097},[364],[296,9099,9102],{"className":9100,"style":9101},[368],"height:0.162em;",[296,9103,9105,9108],{"style":9104},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[296,9106],{"className":9107,"style":377},[376],[296,9109,9111],{"className":9110},[381,382,383,384],[296,9112,9114],{"className":9113},[346,384],[296,9115,7561],{"className":9116},[346,350,384],[296,9118,396],{"className":9119},[395],[296,9121,9123],{"className":9122},[364],[296,9124,9127],{"className":9125,"style":9126},[368],"height:0.2997em;",[296,9128],{},[296,9130],{"className":9131,"style":973},[550],[296,9133,8951],{"className":9134},[572],[296,9136,426],{"className":9137},[499],[296,9139,9141,9144],{"className":9140},[346],[296,9142,2295],{"className":9143},[346,350],[296,9145,9147],{"className":9146},[355],[296,9148,9150,9170],{"className":9149},[359,360],[296,9151,9153,9167],{"className":9152},[364],[296,9154,9156],{"className":9155,"style":7642},[368],[296,9157,9158,9161],{"style":1005},[296,9159],{"className":9160,"style":377},[376],[296,9162,9164],{"className":9163},[381,382,383,384],[296,9165,7561],{"className":9166},[346,350,384],[296,9168,396],{"className":9169},[395],[296,9171,9173],{"className":9172},[364],[296,9174,9176],{"className":9175,"style":403},[368],[296,9177],{},[296,9179,435],{"className":9180},[546],[296,9182,9183,9186],{"style":8023},[296,9184],{"className":9185,"style":590},[376],[296,9187],{"className":9188,"style":8031},[8030],[296,9190,9191,9194],{"style":8034},[296,9192],{"className":9193,"style":590},[376],[296,9195,9197,9200,9203,9243],{"className":9196},[346],[296,9198,8951],{"className":9199},[572],[296,9201,426],{"className":9202},[499],[296,9204,9206,9209],{"className":9205},[346],[296,9207,2295],{"className":9208},[346,350],[296,9210,9212],{"className":9211},[355],[296,9213,9215,9235],{"className":9214},[359,360],[296,9216,9218,9232],{"className":9217},[364],[296,9219,9221],{"className":9220,"style":7642},[368],[296,9222,9223,9226],{"style":1005},[296,9224],{"className":9225,"style":377},[376],[296,9227,9229],{"className":9228},[381,382,383,384],[296,9230,8836],{"className":9231,"style":8879},[346,350,384],[296,9233,396],{"className":9234},[395],[296,9236,9238],{"className":9237},[364],[296,9239,9241],{"className":9240,"style":7726},[368],[296,9242],{},[296,9244,435],{"className":9245},[546],[296,9247,396],{"className":9248},[395],[296,9250,9252],{"className":9251},[364],[296,9253,9256],{"className":9254,"style":9255},[368],"height:0.9857em;",[296,9257],{},[296,9259],{"className":9260},[546,984],[212,9262,9263],{},[247,9264,9265],{},"2. 构造损失函数。",[8692,9267,9269],{"className":8694,"code":9268,"language":8696,"meta":40,"style":40},"# 构造损失函数：喂进合适的数据时，它的梯度就是策略梯度\ndef compute_loss(obs, act, weights):\n    logp = get_policy(obs).log_prob(act)\n    return -(logp * weights).mean()\n",[8664,9270,9271,9276,9281,9286],{"__ignoreMap":40},[296,9272,9273],{"class":4380,"line":8701},[296,9274,9275],{},"# 构造损失函数：喂进合适的数据时，它的梯度就是策略梯度\n",[296,9277,9278],{"class":4380,"line":8707},[296,9279,9280],{},"def compute_loss(obs, act, weights):\n",[296,9282,9283],{"class":4380,"line":8713},[296,9284,9285],{},"    logp = get_policy(obs).log_prob(act)\n",[296,9287,9288],{"class":4380,"line":8719},[296,9289,9290],{},"    return -(logp * weights).mean()\n",[212,9292,9293],{},"这一块为策略梯度算法构造一个「损失」函数。喂进正确的数据时，这个损失的梯度就等于策略梯度。所谓正确的数据，是一组按当前策略行动时收集的（状态、动作、权重）三元组，其中状态-动作对的权重是它所属回合的回报。（后面的小节会说明，权重还可以换成别的值，同样正确。）",[208,9295,9296,9303,9309,9404,9450],{"title":8676},[212,9297,9298,9299,9302],{},"虽然我们把它叫作损失函数，但它",[247,9300,9301],{},"不是","监督学习里通常意义上的那种损失函数。它与标准损失函数有两点主要区别。",[212,9304,9305,9308],{},[247,9306,9307],{},"1. 数据分布依赖参数","。损失函数通常定义在一个与待优化参数无关的固定数据分布上。这里不是这样：数据必须从最新的策略里采样。",[212,9310,9311,9314,9315,9403],{},[247,9312,9313],{},"2. 它不度量性能","。损失函数通常度量我们关心的性能指标。这里我们关心的是期望回报 ",[296,9316,9318,9342],{"className":9317},[299],[296,9319,9321],{"className":9320},[303],[305,9322,9323],{"xmlns":307},[309,9324,9325,9339],{},[312,9326,9327,9329,9331,9337],{},[318,9328,421],{},[423,9330,426],{"stretchy":425},[315,9332,9333,9335],{},[318,9334,320],{},[318,9336,323],{},[423,9338,435],{"stretchy":425},[325,9340,9341],{"encoding":327},"J(\\pi_{\\theta})",[296,9343,9345],{"className":9344,"ariaHidden":333},[332],[296,9346,9348,9351,9354,9357,9400],{"className":9347},[337],[296,9349],{"className":9350,"style":491},[341],[296,9352,421],{"className":9353,"style":495},[346,350],[296,9355,426],{"className":9356},[499],[296,9358,9360,9363],{"className":9359},[346],[296,9361,320],{"className":9362,"style":351},[346,350],[296,9364,9366],{"className":9365},[355],[296,9367,9369,9392],{"className":9368},[359,360],[296,9370,9372,9389],{"className":9371},[364],[296,9373,9375],{"className":9374,"style":369},[368],[296,9376,9377,9380],{"style":372},[296,9378],{"className":9379,"style":377},[376],[296,9381,9383],{"className":9382},[381,382,383,384],[296,9384,9386],{"className":9385},[346,384],[296,9387,323],{"className":9388,"style":391},[346,350,384],[296,9390,396],{"className":9391},[395],[296,9393,9395],{"className":9394},[364],[296,9396,9398],{"className":9397,"style":403},[368],[296,9399],{},[296,9401,435],{"className":9402},[546],"，但这个「损失」函数连在期望意义上都不近似它。这个「损失」函数之所以有用，只是因为：在当前参数处、用当前参数生成的数据来评估时，它的梯度是性能的负梯度。",[212,9405,9406,9407,9411,9412,9449],{},"但走出梯度下降的第一步之后，它就与性能没有任何联系了。这意味着：对给定的一批数据，最小化这个「损失」函数",[9408,9409,9410],"em",{},"完全","不保证提升期望回报。你可以把这个损失压到 ",[296,9413,9415,9433],{"className":9414},[299],[296,9416,9418],{"className":9417},[303],[305,9419,9420],{"xmlns":307},[309,9421,9422,9430],{},[312,9423,9424,9427],{},[423,9425,9426],{},"−",[318,9428,9429],{"mathvariant":802},"∞",[325,9431,9432],{"encoding":327},"-\\infty",[296,9434,9436],{"className":9435,"ariaHidden":333},[332],[296,9437,9439,9443,9446],{"className":9438},[337],[296,9440],{"className":9441,"style":9442},[341],"height:0.6667em;vertical-align:-0.0833em;",[296,9444,9426],{"className":9445},[346],[296,9447,9429],{"className":9448},[346],"，而策略性能照样会崩；实际上通常就会崩。有的深度 RL 研究者会把这种结果描述成策略对这批数据「过拟合」了。这个说法有描述作用，但别按字面理解，因为它与泛化误差无关。",[212,9451,9452],{},"我们特意提这一点，是因为机器学习从业者习惯把损失函数当作训练中的有用信号：「损失降了，一切安好。」在策略梯度里，这个直觉是错的，你只该关心平均回报。损失函数没有任何意义。",[208,9454,9455,9469,9493],{"title":8676},[212,9456,9457,9458,9461,9462,9464,9465,9468],{},"这里构造 ",[8664,9459,9460],{},"logp"," 张量的方式（调用 PyTorch ",[8664,9463,8801],{}," 对象的 ",[8664,9466,9467],{},"log_prob"," 方法）换成其他分布对象时，可能需要一些修改。",[212,9470,9471,9472,9477,9478,9481,9482,9485,9486,9488,9489,9492],{},"比如，如果你用的是 ",[216,9473,9476],{"href":9474,"rel":9475},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Fdistributions.html#normal",[220],"Normal 分布","（对应对角高斯策略），调用 ",[8664,9479,9480],{},"policy.log_prob(act)"," 返回的张量会给每个向量型动作的每个分量各一个对数概率：输入形状 ",[8664,9483,9484],{},"(batch, act_dim)"," 的张量，出来的还是 ",[8664,9487,9484],{},"，而构造 RL 损失需要的是 ",[8664,9490,9491],{},"(batch,)","。这时要把动作分量的对数概率加总，得到动作的对数概率，也就是计算：",[8692,9494,9496],{"className":8694,"code":9495,"language":8696,"meta":40,"style":40},"logp = get_policy(obs).log_prob(act).sum(axis=-1)\n",[8664,9497,9498],{"__ignoreMap":40},[296,9499,9500],{"class":4380,"line":8701},[296,9501,9495],{},[212,9503,9504],{},[247,9505,9506],{},"3. 跑一轮训练。",[8692,9508,9510],{"className":8694,"code":9509,"language":8696,"meta":40,"style":40},"# 用于训练策略\ndef train_one_epoch():\n    # 建几个空列表，用于记录\n    batch_obs = []          # 观测\n    batch_acts = []         # 动作\n    batch_weights = []      # 策略梯度里的 R(tau) 权重\n    batch_rets = []         # 记录回合回报\n    batch_lens = []         # 记录回合长度\n\n    # 重置回合内变量\n    obs = env.reset()       # 第一个观测来自起始分布\n    done = False            # 环境给出的「回合结束」信号\n    ep_rews = []            # 本回合累计的奖励列表\n\n    # 每个 epoch 渲染第一个回合\n    finished_rendering_this_epoch = False\n\n    # 用当前策略在环境中行动，收集经验\n    while True:\n\n        # 渲染\n        if (not finished_rendering_this_epoch) and render:\n            env.render()\n\n        # 保存观测\n        batch_obs.append(obs.copy())\n\n        # 在环境中行动\n        act = get_action(torch.as_tensor(obs, dtype=torch.float32))\n        obs, rew, done, _ = env.step(act)\n\n        # 保存动作与奖励\n        batch_acts.append(act)\n        ep_rews.append(rew)\n\n        if done:\n            # 回合结束，记录该回合的信息\n            ep_ret, ep_len = sum(ep_rews), len(ep_rews)\n            batch_rets.append(ep_ret)\n            batch_lens.append(ep_len)\n\n            # 每个 logprob(a|s) 的权重是 R(tau)\n            batch_weights += [ep_ret] * ep_len\n\n            # 重置回合内变量\n            obs, done, ep_rews = env.reset(), False, []\n\n            # 这个 epoch 不再渲染\n            finished_rendering_this_epoch = True\n\n            # 经验够了就结束收集循环\n            if len(batch_obs) > batch_size:\n                break\n\n    # 做一次策略梯度更新\n    optimizer.zero_grad()\n    batch_loss = compute_loss(obs=torch.as_tensor(batch_obs, dtype=torch.float32),\n                              act=torch.as_tensor(batch_acts, dtype=torch.int32),\n                              weights=torch.as_tensor(batch_weights, dtype=torch.float32)\n                              )\n    batch_loss.backward()\n    optimizer.step()\n    return batch_loss, batch_rets, batch_lens\n",[8664,9511,9512,9517,9522,9527,9532,9537,9542,9547,9552,9556,9561,9566,9572,9578,9583,9589,9595,9600,9606,9612,9617,9623,9629,9635,9640,9646,9652,9657,9663,9669,9675,9680,9686,9692,9698,9703,9709,9715,9721,9727,9733,9738,9744,9750,9755,9761,9767,9772,9778,9784,9789,9795,9801,9807,9812,9818,9824,9830,9836,9842,9848,9854,9860],{"__ignoreMap":40},[296,9513,9514],{"class":4380,"line":8701},[296,9515,9516],{},"# 用于训练策略\n",[296,9518,9519],{"class":4380,"line":8707},[296,9520,9521],{},"def train_one_epoch():\n",[296,9523,9524],{"class":4380,"line":8713},[296,9525,9526],{},"    # 建几个空列表，用于记录\n",[296,9528,9529],{"class":4380,"line":8719},[296,9530,9531],{},"    batch_obs = []          # 观测\n",[296,9533,9534],{"class":4380,"line":8725},[296,9535,9536],{},"    batch_acts = []         # 动作\n",[296,9538,9539],{"class":4380,"line":8731},[296,9540,9541],{},"    batch_weights = []      # 策略梯度里的 R(tau) 权重\n",[296,9543,9544],{"class":4380,"line":8737},[296,9545,9546],{},"    batch_rets = []         # 记录回合回报\n",[296,9548,9549],{"class":4380,"line":8743},[296,9550,9551],{},"    batch_lens = []         # 记录回合长度\n",[296,9553,9554],{"class":4380,"line":8748},[296,9555,8716],{"emptyLinePlaceholder":36},[296,9557,9558],{"class":4380,"line":8754},[296,9559,9560],{},"    # 重置回合内变量\n",[296,9562,9563],{"class":4380,"line":8760},[296,9564,9565],{},"    obs = env.reset()       # 第一个观测来自起始分布\n",[296,9567,9569],{"class":4380,"line":9568},12,[296,9570,9571],{},"    done = False            # 环境给出的「回合结束」信号\n",[296,9573,9575],{"class":4380,"line":9574},13,[296,9576,9577],{},"    ep_rews = []            # 本回合累计的奖励列表\n",[296,9579,9581],{"class":4380,"line":9580},14,[296,9582,8716],{"emptyLinePlaceholder":36},[296,9584,9586],{"class":4380,"line":9585},15,[296,9587,9588],{},"    # 每个 epoch 渲染第一个回合\n",[296,9590,9592],{"class":4380,"line":9591},16,[296,9593,9594],{},"    finished_rendering_this_epoch = False\n",[296,9596,9598],{"class":4380,"line":9597},17,[296,9599,8716],{"emptyLinePlaceholder":36},[296,9601,9603],{"class":4380,"line":9602},18,[296,9604,9605],{},"    # 用当前策略在环境中行动，收集经验\n",[296,9607,9609],{"class":4380,"line":9608},19,[296,9610,9611],{},"    while True:\n",[296,9613,9615],{"class":4380,"line":9614},20,[296,9616,8716],{"emptyLinePlaceholder":36},[296,9618,9620],{"class":4380,"line":9619},21,[296,9621,9622],{},"        # 渲染\n",[296,9624,9626],{"class":4380,"line":9625},22,[296,9627,9628],{},"        if (not finished_rendering_this_epoch) and render:\n",[296,9630,9632],{"class":4380,"line":9631},23,[296,9633,9634],{},"            env.render()\n",[296,9636,9638],{"class":4380,"line":9637},24,[296,9639,8716],{"emptyLinePlaceholder":36},[296,9641,9643],{"class":4380,"line":9642},25,[296,9644,9645],{},"        # 保存观测\n",[296,9647,9649],{"class":4380,"line":9648},26,[296,9650,9651],{},"        batch_obs.append(obs.copy())\n",[296,9653,9655],{"class":4380,"line":9654},27,[296,9656,8716],{"emptyLinePlaceholder":36},[296,9658,9660],{"class":4380,"line":9659},28,[296,9661,9662],{},"        # 在环境中行动\n",[296,9664,9666],{"class":4380,"line":9665},29,[296,9667,9668],{},"        act = get_action(torch.as_tensor(obs, dtype=torch.float32))\n",[296,9670,9672],{"class":4380,"line":9671},30,[296,9673,9674],{},"        obs, rew, done, _ = env.step(act)\n",[296,9676,9678],{"class":4380,"line":9677},31,[296,9679,8716],{"emptyLinePlaceholder":36},[296,9681,9683],{"class":4380,"line":9682},32,[296,9684,9685],{},"        # 保存动作与奖励\n",[296,9687,9689],{"class":4380,"line":9688},33,[296,9690,9691],{},"        batch_acts.append(act)\n",[296,9693,9695],{"class":4380,"line":9694},34,[296,9696,9697],{},"        ep_rews.append(rew)\n",[296,9699,9701],{"class":4380,"line":9700},35,[296,9702,8716],{"emptyLinePlaceholder":36},[296,9704,9706],{"class":4380,"line":9705},36,[296,9707,9708],{},"        if done:\n",[296,9710,9712],{"class":4380,"line":9711},37,[296,9713,9714],{},"            # 回合结束，记录该回合的信息\n",[296,9716,9718],{"class":4380,"line":9717},38,[296,9719,9720],{},"            ep_ret, ep_len = sum(ep_rews), len(ep_rews)\n",[296,9722,9724],{"class":4380,"line":9723},39,[296,9725,9726],{},"            batch_rets.append(ep_ret)\n",[296,9728,9730],{"class":4380,"line":9729},40,[296,9731,9732],{},"            batch_lens.append(ep_len)\n",[296,9734,9736],{"class":4380,"line":9735},41,[296,9737,8716],{"emptyLinePlaceholder":36},[296,9739,9741],{"class":4380,"line":9740},42,[296,9742,9743],{},"            # 每个 logprob(a|s) 的权重是 R(tau)\n",[296,9745,9747],{"class":4380,"line":9746},43,[296,9748,9749],{},"            batch_weights += [ep_ret] * ep_len\n",[296,9751,9753],{"class":4380,"line":9752},44,[296,9754,8716],{"emptyLinePlaceholder":36},[296,9756,9758],{"class":4380,"line":9757},45,[296,9759,9760],{},"            # 重置回合内变量\n",[296,9762,9764],{"class":4380,"line":9763},46,[296,9765,9766],{},"            obs, done, ep_rews = env.reset(), False, []\n",[296,9768,9770],{"class":4380,"line":9769},47,[296,9771,8716],{"emptyLinePlaceholder":36},[296,9773,9775],{"class":4380,"line":9774},48,[296,9776,9777],{},"            # 这个 epoch 不再渲染\n",[296,9779,9781],{"class":4380,"line":9780},49,[296,9782,9783],{},"            finished_rendering_this_epoch = True\n",[296,9785,9787],{"class":4380,"line":9786},50,[296,9788,8716],{"emptyLinePlaceholder":36},[296,9790,9792],{"class":4380,"line":9791},51,[296,9793,9794],{},"            # 经验够了就结束收集循环\n",[296,9796,9798],{"class":4380,"line":9797},52,[296,9799,9800],{},"            if len(batch_obs) > batch_size:\n",[296,9802,9804],{"class":4380,"line":9803},53,[296,9805,9806],{},"                break\n",[296,9808,9810],{"class":4380,"line":9809},54,[296,9811,8716],{"emptyLinePlaceholder":36},[296,9813,9815],{"class":4380,"line":9814},55,[296,9816,9817],{},"    # 做一次策略梯度更新\n",[296,9819,9821],{"class":4380,"line":9820},56,[296,9822,9823],{},"    optimizer.zero_grad()\n",[296,9825,9827],{"class":4380,"line":9826},57,[296,9828,9829],{},"    batch_loss = compute_loss(obs=torch.as_tensor(batch_obs, dtype=torch.float32),\n",[296,9831,9833],{"class":4380,"line":9832},58,[296,9834,9835],{},"                              act=torch.as_tensor(batch_acts, dtype=torch.int32),\n",[296,9837,9839],{"class":4380,"line":9838},59,[296,9840,9841],{},"                              weights=torch.as_tensor(batch_weights, dtype=torch.float32)\n",[296,9843,9845],{"class":4380,"line":9844},60,[296,9846,9847],{},"                              )\n",[296,9849,9851],{"class":4380,"line":9850},61,[296,9852,9853],{},"    batch_loss.backward()\n",[296,9855,9857],{"class":4380,"line":9856},62,[296,9858,9859],{},"    optimizer.step()\n",[296,9861,9863],{"class":4380,"line":9862},63,[296,9864,9865],{},"    return batch_loss, batch_rets, batch_lens\n",[212,9867,9868,9871,9872,8685],{},[8664,9869,9870],{},"train_one_epoch()"," 函数跑一轮（epoch）策略梯度，我们把它定义为：1）经验收集步（源文件第 67 到 102 行），智能体用最新策略在环境中跑若干个回合；2）紧接一次策略梯度更新步（第 104 到 111 行）。算法的主循环就是反复调用 ",[8664,9873,9870],{},[208,9875,9876],{"title":8676},[212,9877,9878],{},"如果你还不熟悉 PyTorch 的优化流程，注意看第 104 到 111 行做一次梯度下降步的模式：先清空梯度缓存；然后计算损失函数；再对损失做一次反向传播，把新梯度累积进梯度缓存；最后让优化器走一步。",[289,9880,9882],{"id":9881},"eglp-引理","EGLP 引理",[212,9884,9885,9886,8685],{},"本小节推导一个在策略梯度理论里到处会用到的中间结果，我们把它叫作",[247,9887,9888],{},"期望 Grad-Log-Prob（EGLP）引理",[212,9890,9891,9893,9894,9969,9970,9998],{},[247,9892,9882],{},"。设 ",[296,9895,9897,9915],{"className":9896},[299],[296,9898,9900],{"className":9899},[303],[305,9901,9902],{"xmlns":307},[309,9903,9904,9912],{},[312,9905,9906],{},[315,9907,9908,9910],{},[318,9909,1671],{},[318,9911,323],{},[325,9913,9914],{"encoding":327},"P_{\\theta}",[296,9916,9918],{"className":9917,"ariaHidden":333},[332],[296,9919,9921,9925],{"className":9920},[337],[296,9922],{"className":9923,"style":9924},[341],"height:0.8333em;vertical-align:-0.15em;",[296,9926,9928,9931],{"className":9927},[346],[296,9929,1671],{"className":9930,"style":1632},[346,350],[296,9932,9934],{"className":9933},[355],[296,9935,9937,9961],{"className":9936},[359,360],[296,9938,9940,9958],{"className":9939},[364],[296,9941,9943],{"className":9942,"style":369},[368],[296,9944,9946,9949],{"style":9945},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[296,9947],{"className":9948,"style":377},[376],[296,9950,9952],{"className":9951},[381,382,383,384],[296,9953,9955],{"className":9954},[346,384],[296,9956,323],{"className":9957,"style":391},[346,350,384],[296,9959,396],{"className":9960},[395],[296,9962,9964],{"className":9963},[364],[296,9965,9967],{"className":9966,"style":403},[368],[296,9968],{}," 是随机变量 ",[296,9971,9973,9986],{"className":9972},[299],[296,9974,9976],{"className":9975},[303],[305,9977,9978],{"xmlns":307},[309,9979,9980,9984],{},[312,9981,9982],{},[318,9983,2295],{},[325,9985,2295],{"encoding":327},[296,9987,9989],{"className":9988,"ariaHidden":333},[332],[296,9990,9992,9995],{"className":9991},[337],[296,9993],{"className":9994,"style":1473},[341],[296,9996,2295],{"className":9997},[346,350]," 上一个带参数的概率分布，那么：",[296,10000,10002],{"className":10001},[751],[296,10003,10005,10069],{"className":10004},[299],[296,10006,10008],{"className":10007},[303],[305,10009,10010],{"xmlns":307,"display":760},[309,10011,10012,10066],{},[312,10013,10014,10034,10062,10064],{},[318,10015,10016],{},[442,10017,10018,10022],{},[423,10019,10020],{},[318,10021,449],{"mathvariant":448},[312,10023,10024,10026,10028],{},[318,10025,2295],{},[423,10027,457],{},[315,10029,10030,10032],{},[318,10031,1671],{},[318,10033,323],{},[312,10035,10036,10038,10044,10046,10048,10054,10056,10058,10060],{},[423,10037,466],{"fence":333},[315,10039,10040,10042],{},[318,10041,803],{"mathvariant":802},[318,10043,323],{},[318,10045,2289],{},[423,10047,2292],{},[315,10049,10050,10052],{},[318,10051,1671],{},[318,10053,323],{},[423,10055,426],{"stretchy":425},[318,10057,2295],{},[423,10059,435],{"stretchy":425},[423,10061,478],{"fence":333},[423,10063,438],{},[778,10065,1425],{},[325,10067,10068],{"encoding":327},"\\underset{x \\sim P_{\\theta}}{\\mathbb{E}}\\left[\\nabla_{\\theta} \\log P_{\\theta}(x)\\right] = 0",[296,10070,10072,10305],{"className":10071,"ariaHidden":333},[332],[296,10073,10075,10079,10178,10181,10296,10299,10302],{"className":10074},[337],[296,10076],{"className":10077,"style":10078},[341],"height:1.6002em;vertical-align:-0.8502em;",[296,10080,10082],{"className":10081},[346],[296,10083,10085],{"className":10084},[572,573],[296,10086,10088,10169],{"className":10087},[359,360],[296,10089,10091,10166],{"className":10090},[364],[296,10092,10094,10156],{"className":10093,"style":583},[368],[296,10095,10097,10100],{"style":10096},"top:-2.3557em;margin-left:0em;",[296,10098],{"className":10099,"style":590},[376],[296,10101,10103],{"className":10102},[381,382,383,384],[296,10104,10106,10109,10112],{"className":10105},[346,384],[296,10107,2295],{"className":10108},[346,350,384],[296,10110,457],{"className":10111},[555,384],[296,10113,10115,10118],{"className":10114},[346,384],[296,10116,1671],{"className":10117,"style":1632},[346,350,384],[296,10119,10121],{"className":10120},[355],[296,10122,10124,10148],{"className":10123},[359,360],[296,10125,10127,10145],{"className":10126},[364],[296,10128,10130],{"className":10129,"style":622},[368],[296,10131,10133,10136],{"style":10132},"top:-2.3488em;margin-left:-0.1389em;margin-right:0.0714em;",[296,10134],{"className":10135,"style":629},[376],[296,10137,10139],{"className":10138},[381,633,634,384],[296,10140,10142],{"className":10141},[346,384],[296,10143,323],{"className":10144,"style":391},[346,350,384],[296,10146,396],{"className":10147},[395],[296,10149,10151],{"className":10150},[364],[296,10152,10154],{"className":10153,"style":650},[368],[296,10155],{},[296,10157,10158,10161],{"style":655},[296,10159],{"className":10160,"style":590},[376],[296,10162,10163],{},[296,10164,449],{"className":10165},[572,664],[296,10167,396],{"className":10168},[395],[296,10170,10172],{"className":10171},[364],[296,10173,10176],{"className":10174,"style":10175},[368],"height:0.8502em;",[296,10177],{},[296,10179],{"className":10180,"style":973},[550],[296,10182,10184,10187,10230,10233,10238,10241,10284,10287,10290,10293],{"className":10183},[977],[296,10185,466],{"className":10186,"style":1085},[499,1084],[296,10188,10190,10193],{"className":10189},[346],[296,10191,803],{"className":10192},[346],[296,10194,10196],{"className":10195},[355],[296,10197,10199,10222],{"className":10198},[359,360],[296,10200,10202,10219],{"className":10201},[364],[296,10203,10205],{"className":10204,"style":369},[368],[296,10206,10207,10210],{"style":1005},[296,10208],{"className":10209,"style":377},[376],[296,10211,10213],{"className":10212},[381,382,383,384],[296,10214,10216],{"className":10215},[346,384],[296,10217,323],{"className":10218,"style":391},[346,350,384],[296,10220,396],{"className":10221},[395],[296,10223,10225],{"className":10224},[364],[296,10226,10228],{"className":10227,"style":403},[368],[296,10229],{},[296,10231],{"className":10232,"style":973},[550],[296,10234,2312,10236],{"className":10235},[572],[296,10237,2316],{"style":2315},[296,10239],{"className":10240,"style":973},[550],[296,10242,10244,10247],{"className":10243},[346],[296,10245,1671],{"className":10246,"style":1632},[346,350],[296,10248,10250],{"className":10249},[355],[296,10251,10253,10276],{"className":10252},[359,360],[296,10254,10256,10273],{"className":10255},[364],[296,10257,10259],{"className":10258,"style":369},[368],[296,10260,10261,10264],{"style":9945},[296,10262],{"className":10263,"style":377},[376],[296,10265,10267],{"className":10266},[381,382,383,384],[296,10268,10270],{"className":10269},[346,384],[296,10271,323],{"className":10272,"style":391},[346,350,384],[296,10274,396],{"className":10275},[395],[296,10277,10279],{"className":10278},[364],[296,10280,10282],{"className":10281,"style":403},[368],[296,10283],{},[296,10285,426],{"className":10286},[499],[296,10288,2295],{"className":10289},[346,350],[296,10291,435],{"className":10292},[546],[296,10294,478],{"className":10295,"style":1085},[546,1084],[296,10297],{"className":10298,"style":551},[550],[296,10300,438],{"className":10301},[555],[296,10303],{"className":10304,"style":551},[550],[296,10306,10308,10312],{"className":10307},[337],[296,10309],{"className":10310,"style":10311},[341],"height:0.6444em;",[296,10313,1425],{"className":10314},[346],[208,10316,10318,10325,10488,10491,10776,10779],{"title":10317},"证明",[212,10319,10320,10321,10324],{},"回忆所有概率分布都是",[9408,10322,10323],{},"归一化","的：",[296,10326,10328],{"className":10327},[751],[296,10329,10331,10365],{"className":10330},[299],[296,10332,10334],{"className":10333},[303],[305,10335,10336],{"xmlns":307,"display":760},[309,10337,10338,10362],{},[312,10339,10340,10346,10352,10354,10356,10358,10360],{},[315,10341,10342,10344],{},[423,10343,5349],{},[318,10345,2295],{},[315,10347,10348,10350],{},[318,10349,1671],{},[318,10351,323],{},[423,10353,426],{"stretchy":425},[318,10355,2295],{},[423,10357,435],{"stretchy":425},[423,10359,438],{},[778,10361,780],{},[325,10363,10364],{"encoding":327},"\\int_x P_{\\theta}(x) = 1",[296,10366,10368,10479],{"className":10367,"ariaHidden":333},[332],[296,10369,10371,10375,10415,10418,10461,10464,10467,10470,10473,10476],{"className":10370},[337],[296,10372],{"className":10373,"style":10374},[341],"height:2.2719em;vertical-align:-0.9119em;",[296,10376,10378,10381],{"className":10377},[572],[296,10379,5349],{"className":10380,"style":6322},[572,1957,1958],[296,10382,10384],{"className":10383},[355],[296,10385,10387,10407],{"className":10386},[359,360],[296,10388,10390,10404],{"className":10389},[364],[296,10391,10393],{"className":10392,"style":6335},[368],[296,10394,10395,10398],{"style":6338},[296,10396],{"className":10397,"style":377},[376],[296,10399,10401],{"className":10400},[381,382,383,384],[296,10402,2295],{"className":10403},[346,350,384],[296,10405,396],{"className":10406},[395],[296,10408,10410],{"className":10409},[364],[296,10411,10413],{"className":10412,"style":6360},[368],[296,10414],{},[296,10416],{"className":10417,"style":973},[550],[296,10419,10421,10424],{"className":10420},[346],[296,10422,1671],{"className":10423,"style":1632},[346,350],[296,10425,10427],{"className":10426},[355],[296,10428,10430,10453],{"className":10429},[359,360],[296,10431,10433,10450],{"className":10432},[364],[296,10434,10436],{"className":10435,"style":369},[368],[296,10437,10438,10441],{"style":9945},[296,10439],{"className":10440,"style":377},[376],[296,10442,10444],{"className":10443},[381,382,383,384],[296,10445,10447],{"className":10446},[346,384],[296,10448,323],{"className":10449,"style":391},[346,350,384],[296,10451,396],{"className":10452},[395],[296,10454,10456],{"className":10455},[364],[296,10457,10459],{"className":10458,"style":403},[368],[296,10460],{},[296,10462,426],{"className":10463},[499],[296,10465,2295],{"className":10466},[346,350],[296,10468,435],{"className":10469},[546],[296,10471],{"className":10472,"style":551},[550],[296,10474,438],{"className":10475},[555],[296,10477],{"className":10478,"style":551},[550],[296,10480,10482,10485],{"className":10481},[337],[296,10483],{"className":10484,"style":10311},[341],[296,10486,780],{"className":10487},[346],[212,10489,10490],{},"对归一化条件两边取梯度：",[296,10492,10494],{"className":10493},[751],[296,10495,10497,10547],{"className":10496},[299],[296,10498,10500],{"className":10499},[303],[305,10501,10502],{"xmlns":307,"display":760},[309,10503,10504,10544],{},[312,10505,10506,10512,10518,10524,10526,10528,10530,10532,10538,10540,10542],{},[315,10507,10508,10510],{},[318,10509,803],{"mathvariant":802},[318,10511,323],{},[315,10513,10514,10516],{},[423,10515,5349],{},[318,10517,2295],{},[315,10519,10520,10522],{},[318,10521,1671],{},[318,10523,323],{},[423,10525,426],{"stretchy":425},[318,10527,2295],{},[423,10529,435],{"stretchy":425},[423,10531,438],{},[315,10533,10534,10536],{},[318,10535,803],{"mathvariant":802},[318,10537,323],{},[778,10539,780],{},[423,10541,438],{},[778,10543,1425],{},[325,10545,10546],{"encoding":327},"\\nabla_{\\theta} \\int_x P_{\\theta}(x) = \\nabla_{\\theta} 1 = 0",[296,10548,10550,10706,10767],{"className":10549,"ariaHidden":333},[332],[296,10551,10553,10556,10599,10602,10642,10645,10688,10691,10694,10697,10700,10703],{"className":10552},[337],[296,10554],{"className":10555,"style":10374},[341],[296,10557,10559,10562],{"className":10558},[346],[296,10560,803],{"className":10561},[346],[296,10563,10565],{"className":10564},[355],[296,10566,10568,10591],{"className":10567},[359,360],[296,10569,10571,10588],{"className":10570},[364],[296,10572,10574],{"className":10573,"style":369},[368],[296,10575,10576,10579],{"style":1005},[296,10577],{"className":10578,"style":377},[376],[296,10580,10582],{"className":10581},[381,382,383,384],[296,10583,10585],{"className":10584},[346,384],[296,10586,323],{"className":10587,"style":391},[346,350,384],[296,10589,396],{"className":10590},[395],[296,10592,10594],{"className":10593},[364],[296,10595,10597],{"className":10596,"style":403},[368],[296,10598],{},[296,10600],{"className":10601,"style":973},[550],[296,10603,10605,10608],{"className":10604},[572],[296,10606,5349],{"className":10607,"style":6322},[572,1957,1958],[296,10609,10611],{"className":10610},[355],[296,10612,10614,10634],{"className":10613},[359,360],[296,10615,10617,10631],{"className":10616},[364],[296,10618,10620],{"className":10619,"style":6335},[368],[296,10621,10622,10625],{"style":6338},[296,10623],{"className":10624,"style":377},[376],[296,10626,10628],{"className":10627},[381,382,383,384],[296,10629,2295],{"className":10630},[346,350,384],[296,10632,396],{"className":10633},[395],[296,10635,10637],{"className":10636},[364],[296,10638,10640],{"className":10639,"style":6360},[368],[296,10641],{},[296,10643],{"className":10644,"style":973},[550],[296,10646,10648,10651],{"className":10647},[346],[296,10649,1671],{"className":10650,"style":1632},[346,350],[296,10652,10654],{"className":10653},[355],[296,10655,10657,10680],{"className":10656},[359,360],[296,10658,10660,10677],{"className":10659},[364],[296,10661,10663],{"className":10662,"style":369},[368],[296,10664,10665,10668],{"style":9945},[296,10666],{"className":10667,"style":377},[376],[296,10669,10671],{"className":10670},[381,382,383,384],[296,10672,10674],{"className":10673},[346,384],[296,10675,323],{"className":10676,"style":391},[346,350,384],[296,10678,396],{"className":10679},[395],[296,10681,10683],{"className":10682},[364],[296,10684,10686],{"className":10685,"style":403},[368],[296,10687],{},[296,10689,426],{"className":10690},[499],[296,10692,2295],{"className":10693},[346,350],[296,10695,435],{"className":10696},[546],[296,10698],{"className":10699,"style":551},[550],[296,10701,438],{"className":10702},[555],[296,10704],{"className":10705,"style":551},[550],[296,10707,10709,10712,10755,10758,10761,10764],{"className":10708},[337],[296,10710],{"className":10711,"style":9924},[341],[296,10713,10715,10718],{"className":10714},[346],[296,10716,803],{"className":10717},[346],[296,10719,10721],{"className":10720},[355],[296,10722,10724,10747],{"className":10723},[359,360],[296,10725,10727,10744],{"className":10726},[364],[296,10728,10730],{"className":10729,"style":369},[368],[296,10731,10732,10735],{"style":1005},[296,10733],{"className":10734,"style":377},[376],[296,10736,10738],{"className":10737},[381,382,383,384],[296,10739,10741],{"className":10740},[346,384],[296,10742,323],{"className":10743,"style":391},[346,350,384],[296,10745,396],{"className":10746},[395],[296,10748,10750],{"className":10749},[364],[296,10751,10753],{"className":10752,"style":403},[368],[296,10754],{},[296,10756,780],{"className":10757},[346],[296,10759],{"className":10760,"style":551},[550],[296,10762,438],{"className":10763},[555],[296,10765],{"className":10766,"style":551},[550],[296,10768,10770,10773],{"className":10769},[337],[296,10771],{"className":10772,"style":10311},[341],[296,10774,1425],{"className":10775},[346],[212,10777,10778],{},"用对数求导技巧得到：",[296,10780,10782],{"className":10781},[751],[296,10783,10785,11009],{"className":10784},[299],[296,10786,10788],{"className":10787},[303],[305,10789,10790],{"xmlns":307,"display":760},[309,10791,10792,11006],{},[3780,10793,10794,10836,10878,10936],{"rowspacing":3782,"columnalign":3783,"columnspacing":3784},[3786,10795,10796,10802],{},[3789,10797,10798],{},[3792,10799,10800],{"scriptlevel":1425,"displaystyle":333},[778,10801,1425],{},[3789,10803,10804],{},[3792,10805,10806],{"scriptlevel":1425,"displaystyle":333},[312,10807,10808,10810,10812,10818,10824,10830,10832,10834],{},[312,10809],{},[423,10811,438],{},[315,10813,10814,10816],{},[318,10815,803],{"mathvariant":802},[318,10817,323],{},[315,10819,10820,10822],{},[423,10821,5349],{},[318,10823,2295],{},[315,10825,10826,10828],{},[318,10827,1671],{},[318,10829,323],{},[423,10831,426],{"stretchy":425},[318,10833,2295],{},[423,10835,435],{"stretchy":425},[3786,10837,10838,10844],{},[3789,10839,10840],{},[3792,10841,10842],{"scriptlevel":1425,"displaystyle":333},[312,10843],{},[3789,10845,10846],{},[3792,10847,10848],{"scriptlevel":1425,"displaystyle":333},[312,10849,10850,10852,10854,10860,10866,10872,10874,10876],{},[312,10851],{},[423,10853,438],{},[315,10855,10856,10858],{},[423,10857,5349],{},[318,10859,2295],{},[315,10861,10862,10864],{},[318,10863,803],{"mathvariant":802},[318,10865,323],{},[315,10867,10868,10870],{},[318,10869,1671],{},[318,10871,323],{},[423,10873,426],{"stretchy":425},[318,10875,2295],{},[423,10877,435],{"stretchy":425},[3786,10879,10880,10886],{},[3789,10881,10882],{},[3792,10883,10884],{"scriptlevel":1425,"displaystyle":333},[312,10885],{},[3789,10887,10888],{},[3792,10889,10890],{"scriptlevel":1425,"displaystyle":333},[312,10891,10892,10894,10896,10902,10908,10910,10912,10914,10920,10922,10924,10930,10932,10934],{},[312,10893],{},[423,10895,438],{},[315,10897,10898,10900],{},[423,10899,5349],{},[318,10901,2295],{},[315,10903,10904,10906],{},[318,10905,1671],{},[318,10907,323],{},[423,10909,426],{"stretchy":425},[318,10911,2295],{},[423,10913,435],{"stretchy":425},[315,10915,10916,10918],{},[318,10917,803],{"mathvariant":802},[318,10919,323],{},[318,10921,2289],{},[423,10923,2292],{},[315,10925,10926,10928],{},[318,10927,1671],{},[318,10929,323],{},[423,10931,426],{"stretchy":425},[318,10933,2295],{},[423,10935,435],{"stretchy":425},[3786,10937,10938,10948],{},[3789,10939,10940],{},[3792,10941,10942],{"scriptlevel":1425,"displaystyle":333},[312,10943,10944,10946],{},[423,10945,5641],{},[778,10947,1425],{},[3789,10949,10950],{},[3792,10951,10952],{"scriptlevel":1425,"displaystyle":333},[312,10953,10954,10956,10958,10978],{},[312,10955],{},[423,10957,438],{},[318,10959,10960],{},[442,10961,10962,10966],{},[423,10963,10964],{},[318,10965,449],{"mathvariant":448},[312,10967,10968,10970,10972],{},[318,10969,2295],{},[423,10971,457],{},[315,10973,10974,10976],{},[318,10975,1671],{},[318,10977,323],{},[312,10979,10980,10982,10988,10990,10992,10998,11000,11002,11004],{},[423,10981,466],{"fence":333},[315,10983,10984,10986],{},[318,10985,803],{"mathvariant":802},[318,10987,323],{},[318,10989,2289],{},[423,10991,2292],{},[315,10993,10994,10996],{},[318,10995,1671],{},[318,10997,323],{},[423,10999,426],{"stretchy":425},[318,11001,2295],{},[423,11003,435],{"stretchy":425},[423,11005,478],{"fence":333},[325,11007,11008],{"encoding":327},"\\begin{aligned}\n0 &= \\nabla_{\\theta} \\int_x P_{\\theta}(x) \\\\\n&= \\int_x \\nabla_{\\theta} P_{\\theta}(x) \\\\\n&= \\int_x P_{\\theta}(x) \\nabla_{\\theta} \\log P_{\\theta}(x) \\\\\n\\therefore 0 &= \\underset{x \\sim P_{\\theta}}{\\mathbb{E}}\\left[\\nabla_{\\theta} \\log P_{\\theta}(x)\\right]\n\\end{aligned}",[296,11010,11012],{"className":11011,"ariaHidden":333},[332],[296,11013,11015,11019],{"className":11014},[337],[296,11016],{"className":11017,"style":11018},[341],"height:9.406em;vertical-align:-4.453em;",[296,11020,11022],{"className":11021},[346],[296,11023,11025,11099],{"className":11024},[3780],[296,11026,11028],{"className":11027},[4052],[296,11029,11031,11090],{"className":11030},[359,360],[296,11032,11034,11087],{"className":11033},[364],[296,11035,11038,11051,11060,11069],{"className":11036,"style":11037},[368],"height:4.953em;",[296,11039,11041,11045],{"style":11040},"top:-6.953em;",[296,11042],{"className":11043,"style":11044},[376],"height:3.36em;",[296,11046,11048],{"className":11047},[346],[296,11049,1425],{"className":11050},[346],[296,11052,11054,11057],{"style":11053},"top:-4.3811em;",[296,11055],{"className":11056,"style":11044},[376],[296,11058],{"className":11059},[346],[296,11061,11063,11066],{"style":11062},"top:-1.8091em;",[296,11064],{"className":11065,"style":11044},[376],[296,11067],{"className":11068},[346],[296,11070,11072,11075],{"style":11071},"top:0.2428em;",[296,11073],{"className":11074,"style":11044},[376],[296,11076,11078,11081,11084],{"className":11077},[346],[296,11079,5641],{"className":11080},[555,5953],[296,11082],{"className":11083,"style":551},[550],[296,11085,1425],{"className":11086},[346],[296,11088,396],{"className":11089},[395],[296,11091,11093],{"className":11092},[364],[296,11094,11097],{"className":11095,"style":11096},[368],"height:4.453em;",[296,11098],{},[296,11100,11102],{"className":11101},[4169],[296,11103,11105,11888],{"className":11104},[359,360],[296,11106,11108,11885],{"className":11107},[364],[296,11109,11111,11272,11430,11651],{"className":11110,"style":11037},[368],[296,11112,11113,11116],{"style":11040},[296,11114],{"className":11115,"style":11044},[376],[296,11117,11119,11122,11125,11128,11131,11174,11177,11217,11220,11263,11266,11269],{"className":11118},[346],[296,11120],{"className":11121},[346],[296,11123],{"className":11124,"style":551},[550],[296,11126,438],{"className":11127},[555],[296,11129],{"className":11130,"style":551},[550],[296,11132,11134,11137],{"className":11133},[346],[296,11135,803],{"className":11136},[346],[296,11138,11140],{"className":11139},[355],[296,11141,11143,11166],{"className":11142},[359,360],[296,11144,11146,11163],{"className":11145},[364],[296,11147,11149],{"className":11148,"style":369},[368],[296,11150,11151,11154],{"style":1005},[296,11152],{"className":11153,"style":377},[376],[296,11155,11157],{"className":11156},[381,382,383,384],[296,11158,11160],{"className":11159},[346,384],[296,11161,323],{"className":11162,"style":391},[346,350,384],[296,11164,396],{"className":11165},[395],[296,11167,11169],{"className":11168},[364],[296,11170,11172],{"className":11171,"style":403},[368],[296,11173],{},[296,11175],{"className":11176,"style":973},[550],[296,11178,11180,11183],{"className":11179},[572],[296,11181,5349],{"className":11182,"style":6322},[572,1957,1958],[296,11184,11186],{"className":11185},[355],[296,11187,11189,11209],{"className":11188},[359,360],[296,11190,11192,11206],{"className":11191},[364],[296,11193,11195],{"className":11194,"style":6335},[368],[296,11196,11197,11200],{"style":6338},[296,11198],{"className":11199,"style":377},[376],[296,11201,11203],{"className":11202},[381,382,383,384],[296,11204,2295],{"className":11205},[346,350,384],[296,11207,396],{"className":11208},[395],[296,11210,11212],{"className":11211},[364],[296,11213,11215],{"className":11214,"style":6360},[368],[296,11216],{},[296,11218],{"className":11219,"style":973},[550],[296,11221,11223,11226],{"className":11222},[346],[296,11224,1671],{"className":11225,"style":1632},[346,350],[296,11227,11229],{"className":11228},[355],[296,11230,11232,11255],{"className":11231},[359,360],[296,11233,11235,11252],{"className":11234},[364],[296,11236,11238],{"className":11237,"style":369},[368],[296,11239,11240,11243],{"style":9945},[296,11241],{"className":11242,"style":377},[376],[296,11244,11246],{"className":11245},[381,382,383,384],[296,11247,11249],{"className":11248},[346,384],[296,11250,323],{"className":11251,"style":391},[346,350,384],[296,11253,396],{"className":11254},[395],[296,11256,11258],{"className":11257},[364],[296,11259,11261],{"className":11260,"style":403},[368],[296,11262],{},[296,11264,426],{"className":11265},[499],[296,11267,2295],{"className":11268},[346,350],[296,11270,435],{"className":11271},[546],[296,11273,11274,11277],{"style":11053},[296,11275],{"className":11276,"style":11044},[376],[296,11278,11280,11283,11286,11289,11292,11332,11335,11378,11421,11424,11427],{"className":11279},[346],[296,11281],{"className":11282},[346],[296,11284],{"className":11285,"style":551},[550],[296,11287,438],{"className":11288},[555],[296,11290],{"className":11291,"style":551},[550],[296,11293,11295,11298],{"className":11294},[572],[296,11296,5349],{"className":11297,"style":6322},[572,1957,1958],[296,11299,11301],{"className":11300},[355],[296,11302,11304,11324],{"className":11303},[359,360],[296,11305,11307,11321],{"className":11306},[364],[296,11308,11310],{"className":11309,"style":6335},[368],[296,11311,11312,11315],{"style":6338},[296,11313],{"className":11314,"style":377},[376],[296,11316,11318],{"className":11317},[381,382,383,384],[296,11319,2295],{"className":11320},[346,350,384],[296,11322,396],{"className":11323},[395],[296,11325,11327],{"className":11326},[364],[296,11328,11330],{"className":11329,"style":6360},[368],[296,11331],{},[296,11333],{"className":11334,"style":973},[550],[296,11336,11338,11341],{"className":11337},[346],[296,11339,803],{"className":11340},[346],[296,11342,11344],{"className":11343},[355],[296,11345,11347,11370],{"className":11346},[359,360],[296,11348,11350,11367],{"className":11349},[364],[296,11351,11353],{"className":11352,"style":369},[368],[296,11354,11355,11358],{"style":1005},[296,11356],{"className":11357,"style":377},[376],[296,11359,11361],{"className":11360},[381,382,383,384],[296,11362,11364],{"className":11363},[346,384],[296,11365,323],{"className":11366,"style":391},[346,350,384],[296,11368,396],{"className":11369},[395],[296,11371,11373],{"className":11372},[364],[296,11374,11376],{"className":11375,"style":403},[368],[296,11377],{},[296,11379,11381,11384],{"className":11380},[346],[296,11382,1671],{"className":11383,"style":1632},[346,350],[296,11385,11387],{"className":11386},[355],[296,11388,11390,11413],{"className":11389},[359,360],[296,11391,11393,11410],{"className":11392},[364],[296,11394,11396],{"className":11395,"style":369},[368],[296,11397,11398,11401],{"style":9945},[296,11399],{"className":11400,"style":377},[376],[296,11402,11404],{"className":11403},[381,382,383,384],[296,11405,11407],{"className":11406},[346,384],[296,11408,323],{"className":11409,"style":391},[346,350,384],[296,11411,396],{"className":11412},[395],[296,11414,11416],{"className":11415},[364],[296,11417,11419],{"className":11418,"style":403},[368],[296,11420],{},[296,11422,426],{"className":11423},[499],[296,11425,2295],{"className":11426},[346,350],[296,11428,435],{"className":11429},[546],[296,11431,11432,11435],{"style":11062},[296,11433],{"className":11434,"style":11044},[376],[296,11436,11438,11441,11444,11447,11450,11490,11493,11536,11539,11542,11545,11588,11591,11596,11599,11642,11645,11648],{"className":11437},[346],[296,11439],{"className":11440},[346],[296,11442],{"className":11443,"style":551},[550],[296,11445,438],{"className":11446},[555],[296,11448],{"className":11449,"style":551},[550],[296,11451,11453,11456],{"className":11452},[572],[296,11454,5349],{"className":11455,"style":6322},[572,1957,1958],[296,11457,11459],{"className":11458},[355],[296,11460,11462,11482],{"className":11461},[359,360],[296,11463,11465,11479],{"className":11464},[364],[296,11466,11468],{"className":11467,"style":6335},[368],[296,11469,11470,11473],{"style":6338},[296,11471],{"className":11472,"style":377},[376],[296,11474,11476],{"className":11475},[381,382,383,384],[296,11477,2295],{"className":11478},[346,350,384],[296,11480,396],{"className":11481},[395],[296,11483,11485],{"className":11484},[364],[296,11486,11488],{"className":11487,"style":6360},[368],[296,11489],{},[296,11491],{"className":11492,"style":973},[550],[296,11494,11496,11499],{"className":11495},[346],[296,11497,1671],{"className":11498,"style":1632},[346,350],[296,11500,11502],{"className":11501},[355],[296,11503,11505,11528],{"className":11504},[359,360],[296,11506,11508,11525],{"className":11507},[364],[296,11509,11511],{"className":11510,"style":369},[368],[296,11512,11513,11516],{"style":9945},[296,11514],{"className":11515,"style":377},[376],[296,11517,11519],{"className":11518},[381,382,383,384],[296,11520,11522],{"className":11521},[346,384],[296,11523,323],{"className":11524,"style":391},[346,350,384],[296,11526,396],{"className":11527},[395],[296,11529,11531],{"className":11530},[364],[296,11532,11534],{"className":11533,"style":403},[368],[296,11535],{},[296,11537,426],{"className":11538},[499],[296,11540,2295],{"className":11541},[346,350],[296,11543,435],{"className":11544},[546],[296,11546,11548,11551],{"className":11547},[346],[296,11549,803],{"className":11550},[346],[296,11552,11554],{"className":11553},[355],[296,11555,11557,11580],{"className":11556},[359,360],[296,11558,11560,11577],{"className":11559},[364],[296,11561,11563],{"className":11562,"style":369},[368],[296,11564,11565,11568],{"style":1005},[296,11566],{"className":11567,"style":377},[376],[296,11569,11571],{"className":11570},[381,382,383,384],[296,11572,11574],{"className":11573},[346,384],[296,11575,323],{"className":11576,"style":391},[346,350,384],[296,11578,396],{"className":11579},[395],[296,11581,11583],{"className":11582},[364],[296,11584,11586],{"className":11585,"style":403},[368],[296,11587],{},[296,11589],{"className":11590,"style":973},[550],[296,11592,2312,11594],{"className":11593},[572],[296,11595,2316],{"style":2315},[296,11597],{"className":11598,"style":973},[550],[296,11600,11602,11605],{"className":11601},[346],[296,11603,1671],{"className":11604,"style":1632},[346,350],[296,11606,11608],{"className":11607},[355],[296,11609,11611,11634],{"className":11610},[359,360],[296,11612,11614,11631],{"className":11613},[364],[296,11615,11617],{"className":11616,"style":369},[368],[296,11618,11619,11622],{"style":9945},[296,11620],{"className":11621,"style":377},[376],[296,11623,11625],{"className":11624},[381,382,383,384],[296,11626,11628],{"className":11627},[346,384],[296,11629,323],{"className":11630,"style":391},[346,350,384],[296,11632,396],{"className":11633},[395],[296,11635,11637],{"className":11636},[364],[296,11638,11640],{"className":11639,"style":403},[368],[296,11641],{},[296,11643,426],{"className":11644},[499],[296,11646,2295],{"className":11647},[346,350],[296,11649,435],{"className":11650},[546],[296,11652,11653,11656],{"style":11071},[296,11654],{"className":11655,"style":11044},[376],[296,11657,11659,11662,11665,11668,11671,11767,11770],{"className":11658},[346],[296,11660],{"className":11661},[346],[296,11663],{"className":11664,"style":551},[550],[296,11666,438],{"className":11667},[555],[296,11669],{"className":11670,"style":551},[550],[296,11672,11674],{"className":11673},[346],[296,11675,11677],{"className":11676},[572,573],[296,11678,11680,11759],{"className":11679},[359,360],[296,11681,11683,11756],{"className":11682},[364],[296,11684,11686,11746],{"className":11685,"style":583},[368],[296,11687,11688,11691],{"style":10096},[296,11689],{"className":11690,"style":590},[376],[296,11692,11694],{"className":11693},[381,382,383,384],[296,11695,11697,11700,11703],{"className":11696},[346,384],[296,11698,2295],{"className":11699},[346,350,384],[296,11701,457],{"className":11702},[555,384],[296,11704,11706,11709],{"className":11705},[346,384],[296,11707,1671],{"className":11708,"style":1632},[346,350,384],[296,11710,11712],{"className":11711},[355],[296,11713,11715,11738],{"className":11714},[359,360],[296,11716,11718,11735],{"className":11717},[364],[296,11719,11721],{"className":11720,"style":622},[368],[296,11722,11723,11726],{"style":10132},[296,11724],{"className":11725,"style":629},[376],[296,11727,11729],{"className":11728},[381,633,634,384],[296,11730,11732],{"className":11731},[346,384],[296,11733,323],{"className":11734,"style":391},[346,350,384],[296,11736,396],{"className":11737},[395],[296,11739,11741],{"className":11740},[364],[296,11742,11744],{"className":11743,"style":650},[368],[296,11745],{},[296,11747,11748,11751],{"style":655},[296,11749],{"className":11750,"style":590},[376],[296,11752,11753],{},[296,11754,449],{"className":11755},[572,664],[296,11757,396],{"className":11758},[395],[296,11760,11762],{"className":11761},[364],[296,11763,11765],{"className":11764,"style":10175},[368],[296,11766],{},[296,11768],{"className":11769,"style":973},[550],[296,11771,11773,11776,11819,11822,11827,11830,11873,11876,11879,11882],{"className":11772},[977],[296,11774,466],{"className":11775,"style":1085},[499,1084],[296,11777,11779,11782],{"className":11778},[346],[296,11780,803],{"className":11781},[346],[296,11783,11785],{"className":11784},[355],[296,11786,11788,11811],{"className":11787},[359,360],[296,11789,11791,11808],{"className":11790},[364],[296,11792,11794],{"className":11793,"style":369},[368],[296,11795,11796,11799],{"style":1005},[296,11797],{"className":11798,"style":377},[376],[296,11800,11802],{"className":11801},[381,382,383,384],[296,11803,11805],{"className":11804},[346,384],[296,11806,323],{"className":11807,"style":391},[346,350,384],[296,11809,396],{"className":11810},[395],[296,11812,11814],{"className":11813},[364],[296,11815,11817],{"className":11816,"style":403},[368],[296,11818],{},[296,11820],{"className":11821,"style":973},[550],[296,11823,2312,11825],{"className":11824},[572],[296,11826,2316],{"style":2315},[296,11828],{"className":11829,"style":973},[550],[296,11831,11833,11836],{"className":11832},[346],[296,11834,1671],{"className":11835,"style":1632},[346,350],[296,11837,11839],{"className":11838},[355],[296,11840,11842,11865],{"className":11841},[359,360],[296,11843,11845,11862],{"className":11844},[364],[296,11846,11848],{"className":11847,"style":369},[368],[296,11849,11850,11853],{"style":9945},[296,11851],{"className":11852,"style":377},[376],[296,11854,11856],{"className":11855},[381,382,383,384],[296,11857,11859],{"className":11858},[346,384],[296,11860,323],{"className":11861,"style":391},[346,350,384],[296,11863,396],{"className":11864},[395],[296,11866,11868],{"className":11867},[364],[296,11869,11871],{"className":11870,"style":403},[368],[296,11872],{},[296,11874,426],{"className":11875},[499],[296,11877,2295],{"className":11878},[346,350],[296,11880,435],{"className":11881},[546],[296,11883,478],{"className":11884,"style":1085},[546,1084],[296,11886,396],{"className":11887},[395],[296,11889,11891],{"className":11890},[364],[296,11892,11894],{"className":11893,"style":11096},[368],[296,11895],{},[208,11897,11899],{"title":11898},"注",[212,11900,11901],{},"本文作者不知道这个引理在文献里是否有标准名字。但它出现得太频繁了，值得给它起个名字，方便引用。",[289,11903,11904],{"id":11904},"别让过去干扰你",[212,11906,11907],{},"看看我们刚才的策略梯度表达式：",[296,11909,11911],{"className":11910},[751],[296,11912,11914,12028],{"className":11913},[299],[296,11915,11917],{"className":11916},[303],[305,11918,11919],{"xmlns":307,"display":760},[309,11920,11921,12025],{},[312,11922,11923,11929,11931,11933,11939,11941,11943,11963],{},[315,11924,11925,11927],{},[318,11926,803],{"mathvariant":802},[318,11928,323],{},[318,11930,421],{},[423,11932,426],{"stretchy":425},[315,11934,11935,11937],{},[318,11936,320],{},[318,11938,323],{},[423,11940,435],{"stretchy":425},[423,11942,438],{},[318,11944,11945],{},[442,11946,11947,11951],{},[423,11948,11949],{},[318,11950,449],{"mathvariant":448},[312,11952,11953,11955,11957],{},[318,11954,454],{},[423,11956,457],{},[315,11958,11959,11961],{},[318,11960,320],{},[318,11962,323],{},[312,11964,11965,11967,11981,11987,11989,11991,11997,11999,12005,12007,12013,12015,12017,12019,12021,12023],{},[423,11966,466],{"fence":333},[1702,11968,11969,11971,11979],{},[423,11970,2704],{},[312,11972,11973,11975,11977],{},[318,11974,1711],{},[423,11976,438],{},[778,11978,1425],{},[318,11980,1454],{},[315,11982,11983,11985],{},[318,11984,803],{"mathvariant":802},[318,11986,323],{},[318,11988,2289],{},[423,11990,2292],{},[315,11992,11993,11995],{},[318,11994,320],{},[318,11996,323],{},[423,11998,426],{"stretchy":425},[315,12000,12001,12003],{},[318,12002,216],{},[318,12004,1711],{},[318,12006,820],{"mathvariant":802},[315,12008,12009,12011],{},[318,12010,1422],{},[318,12012,1711],{},[423,12014,435],{"stretchy":425},[318,12016,469],{},[423,12018,426],{"stretchy":425},[318,12020,454],{},[423,12022,435],{"stretchy":425},[423,12024,478],{"fence":333},[325,12026,12027],{"encoding":327},"\\nabla_{\\theta} J(\\pi_{\\theta}) = \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) R(\\tau)\\right]",[296,12029,12031,12141],{"className":12030,"ariaHidden":333},[332],[296,12032,12034,12037,12080,12083,12086,12129,12132,12135,12138],{"className":12033},[337],[296,12035],{"className":12036,"style":491},[341],[296,12038,12040,12043],{"className":12039},[346],[296,12041,803],{"className":12042},[346],[296,12044,12046],{"className":12045},[355],[296,12047,12049,12072],{"className":12048},[359,360],[296,12050,12052,12069],{"className":12051},[364],[296,12053,12055],{"className":12054,"style":369},[368],[296,12056,12057,12060],{"style":1005},[296,12058],{"className":12059,"style":377},[376],[296,12061,12063],{"className":12062},[381,382,383,384],[296,12064,12066],{"className":12065},[346,384],[296,12067,323],{"className":12068,"style":391},[346,350,384],[296,12070,396],{"className":12071},[395],[296,12073,12075],{"className":12074},[364],[296,12076,12078],{"className":12077,"style":403},[368],[296,12079],{},[296,12081,421],{"className":12082,"style":495},[346,350],[296,12084,426],{"className":12085},[499],[296,12087,12089,12092],{"className":12088},[346],[296,12090,320],{"className":12091,"style":351},[346,350],[296,12093,12095],{"className":12094},[355],[296,12096,12098,12121],{"className":12097},[359,360],[296,12099,12101,12118],{"className":12100},[364],[296,12102,12104],{"className":12103,"style":369},[368],[296,12105,12106,12109],{"style":372},[296,12107],{"className":12108,"style":377},[376],[296,12110,12112],{"className":12111},[381,382,383,384],[296,12113,12115],{"className":12114},[346,384],[296,12116,323],{"className":12117,"style":391},[346,350,384],[296,12119,396],{"className":12120},[395],[296,12122,12124],{"className":12123},[364],[296,12125,12127],{"className":12126,"style":403},[368],[296,12128],{},[296,12130,435],{"className":12131},[546],[296,12133],{"className":12134,"style":551},[550],[296,12136,438],{"className":12137},[555],[296,12139],{"className":12140,"style":551},[550],[296,12142,12144,12147,12243,12246],{"className":12143},[337],[296,12145],{"className":12146,"style":1821},[341],[296,12148,12150],{"className":12149},[346],[296,12151,12153],{"className":12152},[572,573],[296,12154,12156,12235],{"className":12155},[359,360],[296,12157,12159,12232],{"className":12158},[364],[296,12160,12162,12222],{"className":12161,"style":583},[368],[296,12163,12164,12167],{"style":586},[296,12165],{"className":12166,"style":590},[376],[296,12168,12170],{"className":12169},[381,382,383,384],[296,12171,12173,12176,12179],{"className":12172},[346,384],[296,12174,454],{"className":12175,"style":600},[346,350,384],[296,12177,457],{"className":12178},[555,384],[296,12180,12182,12185],{"className":12181},[346,384],[296,12183,320],{"className":12184,"style":351},[346,350,384],[296,12186,12188],{"className":12187},[355],[296,12189,12191,12214],{"className":12190},[359,360],[296,12192,12194,12211],{"className":12193},[364],[296,12195,12197],{"className":12196,"style":622},[368],[296,12198,12199,12202],{"style":625},[296,12200],{"className":12201,"style":629},[376],[296,12203,12205],{"className":12204},[381,633,634,384],[296,12206,12208],{"className":12207},[346,384],[296,12209,323],{"className":12210,"style":391},[346,350,384],[296,12212,396],{"className":12213},[395],[296,12215,12217],{"className":12216},[364],[296,12218,12220],{"className":12219,"style":650},[368],[296,12221],{},[296,12223,12224,12227],{"style":655},[296,12225],{"className":12226,"style":590},[376],[296,12228,12229],{},[296,12230,449],{"className":12231},[572,664],[296,12233,396],{"className":12234},[395],[296,12236,12238],{"className":12237},[364],[296,12239,12241],{"className":12240,"style":674},[368],[296,12242],{},[296,12244],{"className":12245,"style":973},[550],[296,12247,12249,12255,12322,12325,12368,12371,12376,12379,12422,12425,12465,12468,12508,12511,12514,12517,12520,12523],{"className":12248},[977],[296,12250,12252],{"className":12251,"style":1085},[499,1084],[296,12253,466],{"className":12254},[3026,7043],[296,12256,12258],{"className":12257},[572,573],[296,12259,12261,12314],{"className":12260},[359,360],[296,12262,12264,12311],{"className":12263},[364],[296,12265,12267,12287,12297],{"className":12266,"style":1923},[368],[296,12268,12269,12272],{"style":1926},[296,12270],{"className":12271,"style":1930},[376],[296,12273,12275],{"className":12274},[381,382,383,384],[296,12276,12278,12281,12284],{"className":12277},[346,384],[296,12279,1711],{"className":12280},[346,350,384],[296,12282,438],{"className":12283},[555,384],[296,12285,1425],{"className":12286},[346,384],[296,12288,12289,12292],{"style":1948},[296,12290],{"className":12291,"style":1930},[376],[296,12293,12294],{},[296,12295,2704],{"className":12296},[572,1957,1958],[296,12298,12299,12302],{"style":1961},[296,12300],{"className":12301,"style":1930},[376],[296,12303,12305],{"className":12304},[381,382,383,384],[296,12306,12308],{"className":12307},[346,384],[296,12309,1454],{"className":12310,"style":1632},[346,350,384],[296,12312,396],{"className":12313},[395],[296,12315,12317],{"className":12316},[364],[296,12318,12320],{"className":12319,"style":1983},[368],[296,12321],{},[296,12323],{"className":12324,"style":973},[550],[296,12326,12328,12331],{"className":12327},[346],[296,12329,803],{"className":12330},[346],[296,12332,12334],{"className":12333},[355],[296,12335,12337,12360],{"className":12336},[359,360],[296,12338,12340,12357],{"className":12339},[364],[296,12341,12343],{"className":12342,"style":369},[368],[296,12344,12345,12348],{"style":1005},[296,12346],{"className":12347,"style":377},[376],[296,12349,12351],{"className":12350},[381,382,383,384],[296,12352,12354],{"className":12353},[346,384],[296,12355,323],{"className":12356,"style":391},[346,350,384],[296,12358,396],{"className":12359},[395],[296,12361,12363],{"className":12362},[364],[296,12364,12366],{"className":12365,"style":403},[368],[296,12367],{},[296,12369],{"className":12370,"style":973},[550],[296,12372,2312,12374],{"className":12373},[572],[296,12375,2316],{"style":2315},[296,12377],{"className":12378,"style":973},[550],[296,12380,12382,12385],{"className":12381},[346],[296,12383,320],{"className":12384,"style":351},[346,350],[296,12386,12388],{"className":12387},[355],[296,12389,12391,12414],{"className":12390},[359,360],[296,12392,12394,12411],{"className":12393},[364],[296,12395,12397],{"className":12396,"style":369},[368],[296,12398,12399,12402],{"style":372},[296,12400],{"className":12401,"style":377},[376],[296,12403,12405],{"className":12404},[381,382,383,384],[296,12406,12408],{"className":12407},[346,384],[296,12409,323],{"className":12410,"style":391},[346,350,384],[296,12412,396],{"className":12413},[395],[296,12415,12417],{"className":12416},[364],[296,12418,12420],{"className":12419,"style":403},[368],[296,12421],{},[296,12423,426],{"className":12424},[499],[296,12426,12428,12431],{"className":12427},[346],[296,12429,216],{"className":12430},[346,350],[296,12432,12434],{"className":12433},[355],[296,12435,12437,12457],{"className":12436},[359,360],[296,12438,12440,12454],{"className":12439},[364],[296,12441,12443],{"className":12442,"style":2065},[368],[296,12444,12445,12448],{"style":1005},[296,12446],{"className":12447,"style":377},[376],[296,12449,12451],{"className":12450},[381,382,383,384],[296,12452,1711],{"className":12453},[346,350,384],[296,12455,396],{"className":12456},[395],[296,12458,12460],{"className":12459},[364],[296,12461,12463],{"className":12462,"style":403},[368],[296,12464],{},[296,12466,820],{"className":12467},[346],[296,12469,12471,12474],{"className":12470},[346],[296,12472,1422],{"className":12473},[346,350],[296,12475,12477],{"className":12476},[355],[296,12478,12480,12500],{"className":12479},[359,360],[296,12481,12483,12497],{"className":12482},[364],[296,12484,12486],{"className":12485,"style":2065},[368],[296,12487,12488,12491],{"style":1005},[296,12489],{"className":12490,"style":377},[376],[296,12492,12494],{"className":12493},[381,382,383,384],[296,12495,1711],{"className":12496},[346,350,384],[296,12498,396],{"className":12499},[395],[296,12501,12503],{"className":12502},[364],[296,12504,12506],{"className":12505,"style":403},[368],[296,12507],{},[296,12509,435],{"className":12510},[546],[296,12512,469],{"className":12513,"style":683},[346,350],[296,12515,426],{"className":12516},[499],[296,12518,454],{"className":12519,"style":600},[346,350],[296,12521,435],{"className":12522},[546],[296,12524,12526],{"className":12525,"style":1085},[546,1084],[296,12527,478],{"className":12528},[3026,7043],[212,12530,12531,12532,12575,12576,12579],{},"沿这个梯度走一步，会把每个动作的对数概率往上推，推的幅度正比于 ",[296,12533,12535,12554],{"className":12534},[299],[296,12536,12538],{"className":12537},[303],[305,12539,12540],{"xmlns":307},[309,12541,12542,12552],{},[312,12543,12544,12546,12548,12550],{},[318,12545,469],{},[423,12547,426],{"stretchy":425},[318,12549,454],{},[423,12551,435],{"stretchy":425},[325,12553,717],{"encoding":327},[296,12555,12557],{"className":12556,"ariaHidden":333},[332],[296,12558,12560,12563,12566,12569,12572],{"className":12559},[337],[296,12561],{"className":12562,"style":491},[341],[296,12564,469],{"className":12565,"style":683},[346,350],[296,12567,426],{"className":12568},[499],[296,12570,454],{"className":12571,"style":600},[346,350],[296,12573,435],{"className":12574},[546],"，也就是",[9408,12577,12578],{},"曾经拿到的所有奖励","的总和。但这不太讲道理。",[212,12581,12582,12583,12586,12587,12590,12591,12594],{},"智能体强化一个动作，理应只看它的",[9408,12584,12585],{},"后果","。采取动作",[9408,12588,12589],{},"之前","拿到的奖励与这个动作的好坏毫无关系：有关系的只有",[9408,12592,12593],{},"之后","来的奖励。",[212,12596,12597],{},"这个直觉其实藏在数学里：可以证明，策略梯度也能写成",[296,12599,12601],{"className":12600},[751],[296,12602,12604,12776],{"className":12603},[299],[296,12605,12607],{"className":12606},[303],[305,12608,12609],{"xmlns":307,"display":760},[309,12610,12611,12773],{},[312,12612,12613,12619,12621,12623,12629,12631,12633,12653],{},[315,12614,12615,12617],{},[318,12616,803],{"mathvariant":802},[318,12618,323],{},[318,12620,421],{},[423,12622,426],{"stretchy":425},[315,12624,12625,12627],{},[318,12626,320],{},[318,12628,323],{},[423,12630,435],{"stretchy":425},[423,12632,438],{},[318,12634,12635],{},[442,12636,12637,12641],{},[423,12638,12639],{},[318,12640,449],{"mathvariant":448},[312,12642,12643,12645,12647],{},[318,12644,454],{},[423,12646,457],{},[315,12648,12649,12651],{},[318,12650,320],{},[318,12652,323],{},[312,12654,12655,12657,12671,12677,12679,12681,12687,12689,12695,12697,12703,12705,12725,12727,12729,12739,12741,12751,12753,12769,12771],{},[423,12656,466],{"fence":333},[1702,12658,12659,12661,12669],{},[423,12660,2704],{},[312,12662,12663,12665,12667],{},[318,12664,1711],{},[423,12666,438],{},[778,12668,1425],{},[318,12670,1454],{},[315,12672,12673,12675],{},[318,12674,803],{"mathvariant":802},[318,12676,323],{},[318,12678,2289],{},[423,12680,2292],{},[315,12682,12683,12685],{},[318,12684,320],{},[318,12686,323],{},[423,12688,426],{"stretchy":425},[315,12690,12691,12693],{},[318,12692,216],{},[318,12694,1711],{},[318,12696,820],{"mathvariant":802},[315,12698,12699,12701],{},[318,12700,1422],{},[318,12702,1711],{},[423,12704,435],{"stretchy":425},[1702,12706,12707,12709,12723],{},[423,12708,2704],{},[312,12710,12711,12719,12721],{},[12712,12713,12714,12716],"msup",{},[318,12715,1711],{},[423,12717,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},"′",[423,12720,438],{},[318,12722,1711],{},[318,12724,1454],{},[318,12726,469],{},[423,12728,426],{"stretchy":425},[315,12730,12731,12733],{},[318,12732,1422],{},[12712,12734,12735,12737],{},[318,12736,1711],{},[423,12738,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,12740,1428],{"separator":333},[315,12742,12743,12745],{},[318,12744,216],{},[12712,12746,12747,12749],{},[318,12748,1711],{},[423,12750,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,12752,1428],{"separator":333},[315,12754,12755,12757],{},[318,12756,1422],{},[312,12758,12759,12765,12767],{},[12712,12760,12761,12763],{},[318,12762,1711],{},[423,12764,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,12766,776],{},[778,12768,780],{},[423,12770,435],{"stretchy":425},[423,12772,478],{"fence":333},[325,12774,12775],{"encoding":327},"\\nabla_{\\theta} J(\\pi_{\\theta}) = \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) \\sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1})\\right]",[296,12777,12779,12889],{"className":12778,"ariaHidden":333},[332],[296,12780,12782,12785,12828,12831,12834,12877,12880,12883,12886],{"className":12781},[337],[296,12783],{"className":12784,"style":491},[341],[296,12786,12788,12791],{"className":12787},[346],[296,12789,803],{"className":12790},[346],[296,12792,12794],{"className":12793},[355],[296,12795,12797,12820],{"className":12796},[359,360],[296,12798,12800,12817],{"className":12799},[364],[296,12801,12803],{"className":12802,"style":369},[368],[296,12804,12805,12808],{"style":1005},[296,12806],{"className":12807,"style":377},[376],[296,12809,12811],{"className":12810},[381,382,383,384],[296,12812,12814],{"className":12813},[346,384],[296,12815,323],{"className":12816,"style":391},[346,350,384],[296,12818,396],{"className":12819},[395],[296,12821,12823],{"className":12822},[364],[296,12824,12826],{"className":12825,"style":403},[368],[296,12827],{},[296,12829,421],{"className":12830,"style":495},[346,350],[296,12832,426],{"className":12833},[499],[296,12835,12837,12840],{"className":12836},[346],[296,12838,320],{"className":12839,"style":351},[346,350],[296,12841,12843],{"className":12842},[355],[296,12844,12846,12869],{"className":12845},[359,360],[296,12847,12849,12866],{"className":12848},[364],[296,12850,12852],{"className":12851,"style":369},[368],[296,12853,12854,12857],{"style":372},[296,12855],{"className":12856,"style":377},[376],[296,12858,12860],{"className":12859},[381,382,383,384],[296,12861,12863],{"className":12862},[346,384],[296,12864,323],{"className":12865,"style":391},[346,350,384],[296,12867,396],{"className":12868},[395],[296,12870,12872],{"className":12871},[364],[296,12873,12875],{"className":12874,"style":403},[368],[296,12876],{},[296,12878,435],{"className":12879},[546],[296,12881],{"className":12882,"style":551},[550],[296,12884,438],{"className":12885},[555],[296,12887],{"className":12888,"style":551},[550],[296,12890,12892,12896,12992,12995],{"className":12891},[337],[296,12893],{"className":12894,"style":12895},[341],"height:3.1223em;vertical-align:-1.294em;",[296,12897,12899],{"className":12898},[346],[296,12900,12902],{"className":12901},[572,573],[296,12903,12905,12984],{"className":12904},[359,360],[296,12906,12908,12981],{"className":12907},[364],[296,12909,12911,12971],{"className":12910,"style":583},[368],[296,12912,12913,12916],{"style":586},[296,12914],{"className":12915,"style":590},[376],[296,12917,12919],{"className":12918},[381,382,383,384],[296,12920,12922,12925,12928],{"className":12921},[346,384],[296,12923,454],{"className":12924,"style":600},[346,350,384],[296,12926,457],{"className":12927},[555,384],[296,12929,12931,12934],{"className":12930},[346,384],[296,12932,320],{"className":12933,"style":351},[346,350,384],[296,12935,12937],{"className":12936},[355],[296,12938,12940,12963],{"className":12939},[359,360],[296,12941,12943,12960],{"className":12942},[364],[296,12944,12946],{"className":12945,"style":622},[368],[296,12947,12948,12951],{"style":625},[296,12949],{"className":12950,"style":629},[376],[296,12952,12954],{"className":12953},[381,633,634,384],[296,12955,12957],{"className":12956},[346,384],[296,12958,323],{"className":12959,"style":391},[346,350,384],[296,12961,396],{"className":12962},[395],[296,12964,12966],{"className":12965},[364],[296,12967,12969],{"className":12968,"style":650},[368],[296,12970],{},[296,12972,12973,12976],{"style":655},[296,12974],{"className":12975,"style":590},[376],[296,12977,12978],{},[296,12979,449],{"className":12980},[572,664],[296,12982,396],{"className":12983},[395],[296,12985,12987],{"className":12986},[364],[296,12988,12990],{"className":12989,"style":674},[368],[296,12991],{},[296,12993],{"className":12994,"style":973},[550],[296,12996,12998,13004,13071,13074,13117,13120,13125,13128,13171,13174,13214,13217,13257,13260,13263,13360,13363,13366,13369,13442,13445,13448,13520,13523,13526,13604,13607],{"className":12997},[977],[296,12999,13001],{"className":13000,"style":1085},[499,1084],[296,13002,466],{"className":13003},[3026,7043],[296,13005,13007],{"className":13006},[572,573],[296,13008,13010,13063],{"className":13009},[359,360],[296,13011,13013,13060],{"className":13012},[364],[296,13014,13016,13036,13046],{"className":13015,"style":1923},[368],[296,13017,13018,13021],{"style":1926},[296,13019],{"className":13020,"style":1930},[376],[296,13022,13024],{"className":13023},[381,382,383,384],[296,13025,13027,13030,13033],{"className":13026},[346,384],[296,13028,1711],{"className":13029},[346,350,384],[296,13031,438],{"className":13032},[555,384],[296,13034,1425],{"className":13035},[346,384],[296,13037,13038,13041],{"style":1948},[296,13039],{"className":13040,"style":1930},[376],[296,13042,13043],{},[296,13044,2704],{"className":13045},[572,1957,1958],[296,13047,13048,13051],{"style":1961},[296,13049],{"className":13050,"style":1930},[376],[296,13052,13054],{"className":13053},[381,382,383,384],[296,13055,13057],{"className":13056},[346,384],[296,13058,1454],{"className":13059,"style":1632},[346,350,384],[296,13061,396],{"className":13062},[395],[296,13064,13066],{"className":13065},[364],[296,13067,13069],{"className":13068,"style":1983},[368],[296,13070],{},[296,13072],{"className":13073,"style":973},[550],[296,13075,13077,13080],{"className":13076},[346],[296,13078,803],{"className":13079},[346],[296,13081,13083],{"className":13082},[355],[296,13084,13086,13109],{"className":13085},[359,360],[296,13087,13089,13106],{"className":13088},[364],[296,13090,13092],{"className":13091,"style":369},[368],[296,13093,13094,13097],{"style":1005},[296,13095],{"className":13096,"style":377},[376],[296,13098,13100],{"className":13099},[381,382,383,384],[296,13101,13103],{"className":13102},[346,384],[296,13104,323],{"className":13105,"style":391},[346,350,384],[296,13107,396],{"className":13108},[395],[296,13110,13112],{"className":13111},[364],[296,13113,13115],{"className":13114,"style":403},[368],[296,13116],{},[296,13118],{"className":13119,"style":973},[550],[296,13121,2312,13123],{"className":13122},[572],[296,13124,2316],{"style":2315},[296,13126],{"className":13127,"style":973},[550],[296,13129,13131,13134],{"className":13130},[346],[296,13132,320],{"className":13133,"style":351},[346,350],[296,13135,13137],{"className":13136},[355],[296,13138,13140,13163],{"className":13139},[359,360],[296,13141,13143,13160],{"className":13142},[364],[296,13144,13146],{"className":13145,"style":369},[368],[296,13147,13148,13151],{"style":372},[296,13149],{"className":13150,"style":377},[376],[296,13152,13154],{"className":13153},[381,382,383,384],[296,13155,13157],{"className":13156},[346,384],[296,13158,323],{"className":13159,"style":391},[346,350,384],[296,13161,396],{"className":13162},[395],[296,13164,13166],{"className":13165},[364],[296,13167,13169],{"className":13168,"style":403},[368],[296,13170],{},[296,13172,426],{"className":13173},[499],[296,13175,13177,13180],{"className":13176},[346],[296,13178,216],{"className":13179},[346,350],[296,13181,13183],{"className":13182},[355],[296,13184,13186,13206],{"className":13185},[359,360],[296,13187,13189,13203],{"className":13188},[364],[296,13190,13192],{"className":13191,"style":2065},[368],[296,13193,13194,13197],{"style":1005},[296,13195],{"className":13196,"style":377},[376],[296,13198,13200],{"className":13199},[381,382,383,384],[296,13201,1711],{"className":13202},[346,350,384],[296,13204,396],{"className":13205},[395],[296,13207,13209],{"className":13208},[364],[296,13210,13212],{"className":13211,"style":403},[368],[296,13213],{},[296,13215,820],{"className":13216},[346],[296,13218,13220,13223],{"className":13219},[346],[296,13221,1422],{"className":13222},[346,350],[296,13224,13226],{"className":13225},[355],[296,13227,13229,13249],{"className":13228},[359,360],[296,13230,13232,13246],{"className":13231},[364],[296,13233,13235],{"className":13234,"style":2065},[368],[296,13236,13237,13240],{"style":1005},[296,13238],{"className":13239,"style":377},[376],[296,13241,13243],{"className":13242},[381,382,383,384],[296,13244,1711],{"className":13245},[346,350,384],[296,13247,396],{"className":13248},[395],[296,13250,13252],{"className":13251},[364],[296,13253,13255],{"className":13254,"style":403},[368],[296,13256],{},[296,13258,435],{"className":13259},[546],[296,13261],{"className":13262,"style":973},[550],[296,13264,13266],{"className":13265},[572,573],[296,13267,13269,13351],{"className":13268},[359,360],[296,13270,13272,13348],{"className":13271},[364],[296,13273,13275,13327,13337],{"className":13274,"style":1923},[368],[296,13276,13278,13281],{"style":13277},"top:-1.856em;margin-left:0em;",[296,13279],{"className":13280,"style":1930},[376],[296,13282,13284],{"className":13283},[381,382,383,384],[296,13285,13287,13321,13324],{"className":13286},[346,384],[296,13288,13290,13293],{"className":13289},[346,384],[296,13291,1711],{"className":13292},[346,350,384],[296,13294,13296],{"className":13295},[355],[296,13297,13299],{"className":13298},[359],[296,13300,13302],{"className":13301},[364],[296,13303,13306],{"className":13304,"style":13305},[368],"height:0.6828em;",[296,13307,13309,13312],{"style":13308},"top:-2.786em;margin-right:0.0714em;",[296,13310],{"className":13311,"style":629},[376],[296,13313,13315],{"className":13314},[381,633,634,384],[296,13316,13318],{"className":13317},[346,384],[296,13319,12718],{"className":13320},[346,384],[296,13322,438],{"className":13323},[555,384],[296,13325,1711],{"className":13326},[346,350,384],[296,13328,13329,13332],{"style":1948},[296,13330],{"className":13331,"style":1930},[376],[296,13333,13334],{},[296,13335,2704],{"className":13336},[572,1957,1958],[296,13338,13339,13342],{"style":1961},[296,13340],{"className":13341,"style":1930},[376],[296,13343,13345],{"className":13344},[381,382,383,384],[296,13346,1454],{"className":13347,"style":1632},[346,350,384],[296,13349,396],{"className":13350},[395],[296,13352,13354],{"className":13353},[364],[296,13355,13358],{"className":13356,"style":13357},[368],"height:1.294em;",[296,13359],{},[296,13361],{"className":13362,"style":973},[550],[296,13364,469],{"className":13365,"style":683},[346,350],[296,13367,426],{"className":13368},[499],[296,13370,13372,13375],{"className":13371},[346],[296,13373,1422],{"className":13374},[346,350],[296,13376,13378],{"className":13377},[355],[296,13379,13381,13434],{"className":13380},[359,360],[296,13382,13384,13431],{"className":13383},[364],[296,13385,13388],{"className":13386,"style":13387},[368],"height:0.328em;",[296,13389,13390,13393],{"style":1005},[296,13391],{"className":13392,"style":377},[376],[296,13394,13396],{"className":13395},[381,382,383,384],[296,13397,13399],{"className":13398},[346,384],[296,13400,13402,13405],{"className":13401},[346,384],[296,13403,1711],{"className":13404},[346,350,384],[296,13406,13408],{"className":13407},[355],[296,13409,13411],{"className":13410},[359],[296,13412,13414],{"className":13413},[364],[296,13415,13417],{"className":13416,"style":13305},[368],[296,13418,13419,13422],{"style":13308},[296,13420],{"className":13421,"style":629},[376],[296,13423,13425],{"className":13424},[381,633,634,384],[296,13426,13428],{"className":13427},[346,384],[296,13429,12718],{"className":13430},[346,384],[296,13432,396],{"className":13433},[395],[296,13435,13437],{"className":13436},[364],[296,13438,13440],{"className":13439,"style":403},[368],[296,13441],{},[296,13443,1428],{"className":13444},[1539],[296,13446],{"className":13447,"style":973},[550],[296,13449,13451,13454],{"className":13450},[346],[296,13452,216],{"className":13453},[346,350],[296,13455,13457],{"className":13456},[355],[296,13458,13460,13512],{"className":13459},[359,360],[296,13461,13463,13509],{"className":13462},[364],[296,13464,13466],{"className":13465,"style":13387},[368],[296,13467,13468,13471],{"style":1005},[296,13469],{"className":13470,"style":377},[376],[296,13472,13474],{"className":13473},[381,382,383,384],[296,13475,13477],{"className":13476},[346,384],[296,13478,13480,13483],{"className":13479},[346,384],[296,13481,1711],{"className":13482},[346,350,384],[296,13484,13486],{"className":13485},[355],[296,13487,13489],{"className":13488},[359],[296,13490,13492],{"className":13491},[364],[296,13493,13495],{"className":13494,"style":13305},[368],[296,13496,13497,13500],{"style":13308},[296,13498],{"className":13499,"style":629},[376],[296,13501,13503],{"className":13502},[381,633,634,384],[296,13504,13506],{"className":13505},[346,384],[296,13507,12718],{"className":13508},[346,384],[296,13510,396],{"className":13511},[395],[296,13513,13515],{"className":13514},[364],[296,13516,13518],{"className":13517,"style":403},[368],[296,13519],{},[296,13521,1428],{"className":13522},[1539],[296,13524],{"className":13525,"style":973},[550],[296,13527,13529,13532],{"className":13528},[346],[296,13530,1422],{"className":13531},[346,350],[296,13533,13535],{"className":13534},[355],[296,13536,13538,13596],{"className":13537},[359,360],[296,13539,13541,13593],{"className":13540},[364],[296,13542,13544],{"className":13543,"style":13387},[368],[296,13545,13546,13549],{"style":1005},[296,13547],{"className":13548,"style":377},[376],[296,13550,13552],{"className":13551},[381,382,383,384],[296,13553,13555,13587,13590],{"className":13554},[346,384],[296,13556,13558,13561],{"className":13557},[346,384],[296,13559,1711],{"className":13560},[346,350,384],[296,13562,13564],{"className":13563},[355],[296,13565,13567],{"className":13566},[359],[296,13568,13570],{"className":13569},[364],[296,13571,13573],{"className":13572,"style":13305},[368],[296,13574,13575,13578],{"style":13308},[296,13576],{"className":13577,"style":629},[376],[296,13579,13581],{"className":13580},[381,633,634,384],[296,13582,13584],{"className":13583},[346,384],[296,13585,12718],{"className":13586},[346,384],[296,13588,776],{"className":13589},[877,384],[296,13591,780],{"className":13592},[346,384],[296,13594,396],{"className":13595},[395],[296,13597,13599],{"className":13598},[364],[296,13600,13602],{"className":13601,"style":890},[368],[296,13603],{},[296,13605,435],{"className":13606},[546],[296,13608,13610],{"className":13609,"style":1085},[546,1084],[296,13611,478],{"className":13612},[3026,7043],[212,13614,13615,13616,13619],{},"在这个形式里，动作只根据它",[9408,13617,13618],{},"被采取之后","获得的奖励来强化。",[212,13621,13622],{},"我们把这个形式叫作「reward-to-go 策略梯度」，因为轨迹中某一点之后的奖励总和",[296,13624,13626],{"className":13625},[751],[296,13627,13629,13718],{"className":13628},[299],[296,13630,13632],{"className":13631},[303],[305,13633,13634],{"xmlns":307,"display":760},[309,13635,13636,13715],{},[312,13637,13638,13648,13651,13669,13671,13673,13683,13685,13695,13697,13713],{},[315,13639,13640,13646],{},[7819,13641,13642,13644],{"accent":333},[318,13643,469],{},[423,13645,7825],{},[318,13647,1711],{},[423,13649,13650],{},"≐",[1702,13652,13653,13655,13667],{},[423,13654,2704],{},[312,13656,13657,13663,13665],{},[12712,13658,13659,13661],{},[318,13660,1711],{},[423,13662,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,13664,438],{},[318,13666,1711],{},[318,13668,1454],{},[318,13670,469],{},[423,13672,426],{"stretchy":425},[315,13674,13675,13677],{},[318,13676,1422],{},[12712,13678,13679,13681],{},[318,13680,1711],{},[423,13682,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,13684,1428],{"separator":333},[315,13686,13687,13689],{},[318,13688,216],{},[12712,13690,13691,13693],{},[318,13692,1711],{},[423,13694,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,13696,1428],{"separator":333},[315,13698,13699,13701],{},[318,13700,1422],{},[312,13702,13703,13709,13711],{},[12712,13704,13705,13707],{},[318,13706,1711],{},[423,13708,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,13710,776],{},[778,13712,780],{},[423,13714,435],{"stretchy":425},[325,13716,13717],{"encoding":327},"\\hat{R}_t \\doteq \\sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1})",[296,13719,13721,13809],{"className":13720,"ariaHidden":333},[332],[296,13722,13724,13728,13800,13803,13806],{"className":13723},[337],[296,13725],{"className":13726,"style":13727},[341],"height:1.0968em;vertical-align:-0.15em;",[296,13729,13731,13765],{"className":13730},[346],[296,13732,13734],{"className":13733},[346,7925],[296,13735,13737],{"className":13736},[359],[296,13738,13740],{"className":13739},[364],[296,13741,13744,13752],{"className":13742,"style":13743},[368],"height:0.9468em;",[296,13745,13746,13749],{"style":655},[296,13747],{"className":13748,"style":590},[376],[296,13750,469],{"className":13751,"style":683},[346,350],[296,13753,13755,13758],{"style":13754},"top:-3.2523em;",[296,13756],{"className":13757,"style":590},[376],[296,13759,13762],{"className":13760,"style":13761},[7951],"left:-0.1667em;",[296,13763,7825],{"className":13764},[346],[296,13766,13768],{"className":13767},[355],[296,13769,13771,13792],{"className":13770},[359,360],[296,13772,13774,13789],{"className":13773},[364],[296,13775,13777],{"className":13776,"style":2065},[368],[296,13778,13780,13783],{"style":13779},"top:-2.55em;margin-left:-0.0077em;margin-right:0.05em;",[296,13781],{"className":13782,"style":377},[376],[296,13784,13786],{"className":13785},[381,382,383,384],[296,13787,1711],{"className":13788},[346,350,384],[296,13790,396],{"className":13791},[395],[296,13793,13795],{"className":13794},[364],[296,13796,13798],{"className":13797,"style":403},[368],[296,13799],{},[296,13801],{"className":13802,"style":551},[550],[296,13804,13650],{"className":13805},[555],[296,13807],{"className":13808,"style":551},[550],[296,13810,13812,13815,13908,13911,13914,13917,13989,13992,13995,14067,14070,14073,14151],{"className":13811},[337],[296,13813],{"className":13814,"style":12895},[341],[296,13816,13818],{"className":13817},[572,573],[296,13819,13821,13900],{"className":13820},[359,360],[296,13822,13824,13897],{"className":13823},[364],[296,13825,13827,13876,13886],{"className":13826,"style":1923},[368],[296,13828,13829,13832],{"style":13277},[296,13830],{"className":13831,"style":1930},[376],[296,13833,13835],{"className":13834},[381,382,383,384],[296,13836,13838,13870,13873],{"className":13837},[346,384],[296,13839,13841,13844],{"className":13840},[346,384],[296,13842,1711],{"className":13843},[346,350,384],[296,13845,13847],{"className":13846},[355],[296,13848,13850],{"className":13849},[359],[296,13851,13853],{"className":13852},[364],[296,13854,13856],{"className":13855,"style":13305},[368],[296,13857,13858,13861],{"style":13308},[296,13859],{"className":13860,"style":629},[376],[296,13862,13864],{"className":13863},[381,633,634,384],[296,13865,13867],{"className":13866},[346,384],[296,13868,12718],{"className":13869},[346,384],[296,13871,438],{"className":13872},[555,384],[296,13874,1711],{"className":13875},[346,350,384],[296,13877,13878,13881],{"style":1948},[296,13879],{"className":13880,"style":1930},[376],[296,13882,13883],{},[296,13884,2704],{"className":13885},[572,1957,1958],[296,13887,13888,13891],{"style":1961},[296,13889],{"className":13890,"style":1930},[376],[296,13892,13894],{"className":13893},[381,382,383,384],[296,13895,1454],{"className":13896,"style":1632},[346,350,384],[296,13898,396],{"className":13899},[395],[296,13901,13903],{"className":13902},[364],[296,13904,13906],{"className":13905,"style":13357},[368],[296,13907],{},[296,13909],{"className":13910,"style":973},[550],[296,13912,469],{"className":13913,"style":683},[346,350],[296,13915,426],{"className":13916},[499],[296,13918,13920,13923],{"className":13919},[346],[296,13921,1422],{"className":13922},[346,350],[296,13924,13926],{"className":13925},[355],[296,13927,13929,13981],{"className":13928},[359,360],[296,13930,13932,13978],{"className":13931},[364],[296,13933,13935],{"className":13934,"style":13387},[368],[296,13936,13937,13940],{"style":1005},[296,13938],{"className":13939,"style":377},[376],[296,13941,13943],{"className":13942},[381,382,383,384],[296,13944,13946],{"className":13945},[346,384],[296,13947,13949,13952],{"className":13948},[346,384],[296,13950,1711],{"className":13951},[346,350,384],[296,13953,13955],{"className":13954},[355],[296,13956,13958],{"className":13957},[359],[296,13959,13961],{"className":13960},[364],[296,13962,13964],{"className":13963,"style":13305},[368],[296,13965,13966,13969],{"style":13308},[296,13967],{"className":13968,"style":629},[376],[296,13970,13972],{"className":13971},[381,633,634,384],[296,13973,13975],{"className":13974},[346,384],[296,13976,12718],{"className":13977},[346,384],[296,13979,396],{"className":13980},[395],[296,13982,13984],{"className":13983},[364],[296,13985,13987],{"className":13986,"style":403},[368],[296,13988],{},[296,13990,1428],{"className":13991},[1539],[296,13993],{"className":13994,"style":973},[550],[296,13996,13998,14001],{"className":13997},[346],[296,13999,216],{"className":14000},[346,350],[296,14002,14004],{"className":14003},[355],[296,14005,14007,14059],{"className":14006},[359,360],[296,14008,14010,14056],{"className":14009},[364],[296,14011,14013],{"className":14012,"style":13387},[368],[296,14014,14015,14018],{"style":1005},[296,14016],{"className":14017,"style":377},[376],[296,14019,14021],{"className":14020},[381,382,383,384],[296,14022,14024],{"className":14023},[346,384],[296,14025,14027,14030],{"className":14026},[346,384],[296,14028,1711],{"className":14029},[346,350,384],[296,14031,14033],{"className":14032},[355],[296,14034,14036],{"className":14035},[359],[296,14037,14039],{"className":14038},[364],[296,14040,14042],{"className":14041,"style":13305},[368],[296,14043,14044,14047],{"style":13308},[296,14045],{"className":14046,"style":629},[376],[296,14048,14050],{"className":14049},[381,633,634,384],[296,14051,14053],{"className":14052},[346,384],[296,14054,12718],{"className":14055},[346,384],[296,14057,396],{"className":14058},[395],[296,14060,14062],{"className":14061},[364],[296,14063,14065],{"className":14064,"style":403},[368],[296,14066],{},[296,14068,1428],{"className":14069},[1539],[296,14071],{"className":14072,"style":973},[550],[296,14074,14076,14079],{"className":14075},[346],[296,14077,1422],{"className":14078},[346,350],[296,14080,14082],{"className":14081},[355],[296,14083,14085,14143],{"className":14084},[359,360],[296,14086,14088,14140],{"className":14087},[364],[296,14089,14091],{"className":14090,"style":13387},[368],[296,14092,14093,14096],{"style":1005},[296,14094],{"className":14095,"style":377},[376],[296,14097,14099],{"className":14098},[381,382,383,384],[296,14100,14102,14134,14137],{"className":14101},[346,384],[296,14103,14105,14108],{"className":14104},[346,384],[296,14106,1711],{"className":14107},[346,350,384],[296,14109,14111],{"className":14110},[355],[296,14112,14114],{"className":14113},[359],[296,14115,14117],{"className":14116},[364],[296,14118,14120],{"className":14119,"style":13305},[368],[296,14121,14122,14125],{"style":13308},[296,14123],{"className":14124,"style":629},[376],[296,14126,14128],{"className":14127},[381,633,634,384],[296,14129,14131],{"className":14130},[346,384],[296,14132,12718],{"className":14133},[346,384],[296,14135,776],{"className":14136},[877,384],[296,14138,780],{"className":14139},[346,384],[296,14141,396],{"className":14142},[395],[296,14144,14146],{"className":14145},[364],[296,14147,14149],{"className":14148,"style":890},[368],[296,14150],{},[296,14152,435],{"className":14153},[546],[212,14155,14156,14157,14160],{},"就叫作从该点起的 ",[247,14158,14159],{},"reward-to-go","（只往后算的奖励），而这个策略梯度表达式依赖的正是各状态-动作对的 reward-to-go。",[208,14162,14163],{"title":8676},[212,14164,14165,14168],{},[247,14166,14167],{},"但这为什么更好","？策略梯度的一个关键问题是：要把它的样本估计做到低方差，需要多少条采样轨迹。我们出发的那个公式里，强化动作的项正比于过去的奖励，这些项均值为零、方差非零，只会给策略梯度的样本估计添噪音。去掉它们，所需的采样轨迹就少了。",[212,14170,14171,14172,14176],{},"这个论断的（选读）证明见",[216,14173,8684],{"href":14174,"rel":14175},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Fextra_pg_proof1.html",[220],"，最终靠的是 EGLP 引理。",[289,14178,14180],{"id":14179},"实现-reward-to-go-策略梯度","实现 reward-to-go 策略梯度",[212,14182,14183,14184,8667,14187,8490],{},"reward-to-go 策略梯度的 PyTorch 短实现放在 ",[8664,14185,14186],{},"spinup\u002Fexamples\u002Fpytorch\u002Fpg_math\u002F2_rtg_pg.py",[216,14188,8672],{"href":14189,"rel":14190},"https:\u002F\u002Fgithub.com\u002Fopenai\u002Fspinningup\u002Fblob\u002Fmaster\u002Fspinup\u002Fexamples\u002Fpytorch\u002Fpg_math\u002F2_rtg_pg.py",[220],[212,14192,14193,14194,14197],{},"相比 ",[8664,14195,14196],{},"1_simple_pg.py","，唯一的变化是损失函数里用了不同的权重。代码改动非常小：加一个新函数，再改两行。新函数是：",[8692,14199,14201],{"className":8694,"code":14200,"language":8696,"meta":40,"style":40},"def reward_to_go(rews):\n    n = len(rews)\n    rtgs = np.zeros_like(rews)\n    for i in reversed(range(n)):\n        rtgs[i] = rews[i] + (rtgs[i+1] if i+1 \u003C n else 0)\n    return rtgs\n",[8664,14202,14203,14208,14213,14218,14223,14228],{"__ignoreMap":40},[296,14204,14205],{"class":4380,"line":8701},[296,14206,14207],{},"def reward_to_go(rews):\n",[296,14209,14210],{"class":4380,"line":8707},[296,14211,14212],{},"    n = len(rews)\n",[296,14214,14215],{"class":4380,"line":8713},[296,14216,14217],{},"    rtgs = np.zeros_like(rews)\n",[296,14219,14220],{"class":4380,"line":8719},[296,14221,14222],{},"    for i in reversed(range(n)):\n",[296,14224,14225],{"class":4380,"line":8725},[296,14226,14227],{},"        rtgs[i] = rews[i] + (rtgs[i+1] if i+1 \u003C n else 0)\n",[296,14229,14230],{"class":4380,"line":8731},[296,14231,14232],{},"    return rtgs\n",[212,14234,14235],{},"然后把原来的第 91、92 行从：",[8692,14237,14239],{"className":8694,"code":14238,"language":8696,"meta":40,"style":40},"                # 每个 logprob(a|s) 的权重是 R(tau)\n                batch_weights += [ep_ret] * ep_len\n",[8664,14240,14241,14246],{"__ignoreMap":40},[296,14242,14243],{"class":4380,"line":8701},[296,14244,14245],{},"                # 每个 logprob(a|s) 的权重是 R(tau)\n",[296,14247,14248],{"class":4380,"line":8707},[296,14249,14250],{},"                batch_weights += [ep_ret] * ep_len\n",[212,14252,14253],{},"改成：",[8692,14255,14257],{"className":8694,"code":14256,"language":8696,"meta":40,"style":40},"                # 每个 logprob(a_t|s_t) 的权重是从 t 时刻起的 reward-to-go\n                batch_weights += list(reward_to_go(ep_rews))\n",[8664,14258,14259,14264],{"__ignoreMap":40},[296,14260,14261],{"class":4380,"line":8701},[296,14262,14263],{},"                # 每个 logprob(a_t|s_t) 的权重是从 t 时刻起的 reward-to-go\n",[296,14265,14266],{"class":4380,"line":8707},[296,14267,14268],{},"                batch_weights += list(reward_to_go(ep_rews))\n",[208,14270,14271],{"title":8676},[212,14272,8679,14273,8685],{},[216,14274,8684],{"href":14275,"rel":14276},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Fextra_tf_pg_implementation.html#implementing-reward-to-go-policy-gradient",[220],[289,14278,14279],{"id":14279},"策略梯度中的基线",[212,14281,14282,14283,14312],{},"EGLP 引理的一个直接推论：对任何只依赖状态的函数 ",[296,14284,14286,14300],{"className":14285},[299],[296,14287,14289],{"className":14288},[303],[305,14290,14291],{"xmlns":307},[309,14292,14293,14298],{},[312,14294,14295],{},[318,14296,14297],{},"b",[325,14299,14297],{"encoding":327},[296,14301,14303],{"className":14302,"ariaHidden":333},[332],[296,14304,14306,14309],{"className":14305},[337],[296,14307],{"className":14308,"style":3379},[341],[296,14310,14297],{"className":14311},[346,350],"，有",[296,14314,14316],{"className":14315},[751],[296,14317,14319,14411],{"className":14318},[299],[296,14320,14322],{"className":14321},[303],[305,14323,14324],{"xmlns":307,"display":760},[309,14325,14326,14408],{},[312,14327,14328,14352,14404,14406],{},[318,14329,14330],{},[442,14331,14332,14336],{},[423,14333,14334],{},[318,14335,449],{"mathvariant":448},[312,14337,14338,14344,14346],{},[315,14339,14340,14342],{},[318,14341,216],{},[318,14343,1711],{},[423,14345,457],{},[315,14347,14348,14350],{},[318,14349,320],{},[318,14351,323],{},[312,14353,14354,14356,14362,14364,14366,14372,14374,14380,14382,14388,14390,14392,14394,14400,14402],{},[423,14355,466],{"fence":333},[315,14357,14358,14360],{},[318,14359,803],{"mathvariant":802},[318,14361,323],{},[318,14363,2289],{},[423,14365,2292],{},[315,14367,14368,14370],{},[318,14369,320],{},[318,14371,323],{},[423,14373,426],{"stretchy":425},[315,14375,14376,14378],{},[318,14377,216],{},[318,14379,1711],{},[318,14381,820],{"mathvariant":802},[315,14383,14384,14386],{},[318,14385,1422],{},[318,14387,1711],{},[423,14389,435],{"stretchy":425},[318,14391,14297],{},[423,14393,426],{"stretchy":425},[315,14395,14396,14398],{},[318,14397,1422],{},[318,14399,1711],{},[423,14401,435],{"stretchy":425},[423,14403,478],{"fence":333},[423,14405,438],{},[778,14407,1425],{},[325,14409,14410],{"encoding":327},"\\underset{a_t \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\nabla_{\\theta} \\log \\pi_{\\theta}(a_t|s_t) b(s_t)\\right] = 0",[296,14412,14414,14812],{"className":14413,"ariaHidden":333},[332],[296,14415,14417,14420,14556,14559,14803,14806,14809],{"className":14416},[337],[296,14418],{"className":14419,"style":565},[341],[296,14421,14423],{"className":14422},[346],[296,14424,14426],{"className":14425},[572,573],[296,14427,14429,14548],{"className":14428},[359,360],[296,14430,14432,14545],{"className":14431},[364],[296,14433,14435,14535],{"className":14434,"style":583},[368],[296,14436,14437,14440],{"style":586},[296,14438],{"className":14439,"style":590},[376],[296,14441,14443],{"className":14442},[381,382,383,384],[296,14444,14446,14489,14492],{"className":14445},[346,384],[296,14447,14449,14452],{"className":14448},[346,384],[296,14450,216],{"className":14451},[346,350,384],[296,14453,14455],{"className":14454},[355],[296,14456,14458,14480],{"className":14457},[359,360],[296,14459,14461,14477],{"className":14460},[364],[296,14462,14465],{"className":14463,"style":14464},[368],"height:0.2963em;",[296,14466,14468,14471],{"style":14467},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[296,14469],{"className":14470,"style":629},[376],[296,14472,14474],{"className":14473},[381,633,634,384],[296,14475,1711],{"className":14476},[346,350,384],[296,14478,396],{"className":14479},[395],[296,14481,14483],{"className":14482},[364],[296,14484,14487],{"className":14485,"style":14486},[368],"height:0.143em;",[296,14488],{},[296,14490,457],{"className":14491},[555,384],[296,14493,14495,14498],{"className":14494},[346,384],[296,14496,320],{"className":14497,"style":351},[346,350,384],[296,14499,14501],{"className":14500},[355],[296,14502,14504,14527],{"className":14503},[359,360],[296,14505,14507,14524],{"className":14506},[364],[296,14508,14510],{"className":14509,"style":622},[368],[296,14511,14512,14515],{"style":625},[296,14513],{"className":14514,"style":629},[376],[296,14516,14518],{"className":14517},[381,633,634,384],[296,14519,14521],{"className":14520},[346,384],[296,14522,323],{"className":14523,"style":391},[346,350,384],[296,14525,396],{"className":14526},[395],[296,14528,14530],{"className":14529},[364],[296,14531,14533],{"className":14532,"style":650},[368],[296,14534],{},[296,14536,14537,14540],{"style":655},[296,14538],{"className":14539,"style":590},[376],[296,14541,14542],{},[296,14543,449],{"className":14544},[572,664],[296,14546,396],{"className":14547},[395],[296,14549,14551],{"className":14550},[364],[296,14552,14554],{"className":14553,"style":674},[368],[296,14555],{},[296,14557],{"className":14558,"style":973},[550],[296,14560,14562,14565,14608,14611,14616,14619,14662,14665,14705,14708,14748,14751,14754,14757,14797,14800],{"className":14561},[977],[296,14563,466],{"className":14564,"style":1085},[499,1084],[296,14566,14568,14571],{"className":14567},[346],[296,14569,803],{"className":14570},[346],[296,14572,14574],{"className":14573},[355],[296,14575,14577,14600],{"className":14576},[359,360],[296,14578,14580,14597],{"className":14579},[364],[296,14581,14583],{"className":14582,"style":369},[368],[296,14584,14585,14588],{"style":1005},[296,14586],{"className":14587,"style":377},[376],[296,14589,14591],{"className":14590},[381,382,383,384],[296,14592,14594],{"className":14593},[346,384],[296,14595,323],{"className":14596,"style":391},[346,350,384],[296,14598,396],{"className":14599},[395],[296,14601,14603],{"className":14602},[364],[296,14604,14606],{"className":14605,"style":403},[368],[296,14607],{},[296,14609],{"className":14610,"style":973},[550],[296,14612,2312,14614],{"className":14613},[572],[296,14615,2316],{"style":2315},[296,14617],{"className":14618,"style":973},[550],[296,14620,14622,14625],{"className":14621},[346],[296,14623,320],{"className":14624,"style":351},[346,350],[296,14626,14628],{"className":14627},[355],[296,14629,14631,14654],{"className":14630},[359,360],[296,14632,14634,14651],{"className":14633},[364],[296,14635,14637],{"className":14636,"style":369},[368],[296,14638,14639,14642],{"style":372},[296,14640],{"className":14641,"style":377},[376],[296,14643,14645],{"className":14644},[381,382,383,384],[296,14646,14648],{"className":14647},[346,384],[296,14649,323],{"className":14650,"style":391},[346,350,384],[296,14652,396],{"className":14653},[395],[296,14655,14657],{"className":14656},[364],[296,14658,14660],{"className":14659,"style":403},[368],[296,14661],{},[296,14663,426],{"className":14664},[499],[296,14666,14668,14671],{"className":14667},[346],[296,14669,216],{"className":14670},[346,350],[296,14672,14674],{"className":14673},[355],[296,14675,14677,14697],{"className":14676},[359,360],[296,14678,14680,14694],{"className":14679},[364],[296,14681,14683],{"className":14682,"style":2065},[368],[296,14684,14685,14688],{"style":1005},[296,14686],{"className":14687,"style":377},[376],[296,14689,14691],{"className":14690},[381,382,383,384],[296,14692,1711],{"className":14693},[346,350,384],[296,14695,396],{"className":14696},[395],[296,14698,14700],{"className":14699},[364],[296,14701,14703],{"className":14702,"style":403},[368],[296,14704],{},[296,14706,820],{"className":14707},[346],[296,14709,14711,14714],{"className":14710},[346],[296,14712,1422],{"className":14713},[346,350],[296,14715,14717],{"className":14716},[355],[296,14718,14720,14740],{"className":14719},[359,360],[296,14721,14723,14737],{"className":14722},[364],[296,14724,14726],{"className":14725,"style":2065},[368],[296,14727,14728,14731],{"style":1005},[296,14729],{"className":14730,"style":377},[376],[296,14732,14734],{"className":14733},[381,382,383,384],[296,14735,1711],{"className":14736},[346,350,384],[296,14738,396],{"className":14739},[395],[296,14741,14743],{"className":14742},[364],[296,14744,14746],{"className":14745,"style":403},[368],[296,14747],{},[296,14749,435],{"className":14750},[546],[296,14752,14297],{"className":14753},[346,350],[296,14755,426],{"className":14756},[499],[296,14758,14760,14763],{"className":14759},[346],[296,14761,1422],{"className":14762},[346,350],[296,14764,14766],{"className":14765},[355],[296,14767,14769,14789],{"className":14768},[359,360],[296,14770,14772,14786],{"className":14771},[364],[296,14773,14775],{"className":14774,"style":2065},[368],[296,14776,14777,14780],{"style":1005},[296,14778],{"className":14779,"style":377},[376],[296,14781,14783],{"className":14782},[381,382,383,384],[296,14784,1711],{"className":14785},[346,350,384],[296,14787,396],{"className":14788},[395],[296,14790,14792],{"className":14791},[364],[296,14793,14795],{"className":14794,"style":403},[368],[296,14796],{},[296,14798,435],{"className":14799},[546],[296,14801,478],{"className":14802,"style":1085},[546,1084],[296,14804],{"className":14805,"style":551},[550],[296,14807,438],{"className":14808},[555],[296,14810],{"className":14811,"style":551},[550],[296,14813,14815,14818],{"className":14814},[337],[296,14816],{"className":14817,"style":10311},[341],[296,14819,1425],{"className":14820},[346],[212,14822,14823],{},"这让我们可以在策略梯度表达式里加上或减去任意多个这样的项，而期望不变：",[296,14825,14827],{"className":14826},[751],[296,14828,14830,15020],{"className":14829},[299],[296,14831,14833],{"className":14832},[303],[305,14834,14835],{"xmlns":307,"display":760},[309,14836,14837,15017],{},[312,14838,14839,14845,14847,14849,14855,14857,14859,14879],{},[315,14840,14841,14843],{},[318,14842,803],{"mathvariant":802},[318,14844,323],{},[318,14846,421],{},[423,14848,426],{"stretchy":425},[315,14850,14851,14853],{},[318,14852,320],{},[318,14854,323],{},[423,14856,435],{"stretchy":425},[423,14858,438],{},[318,14860,14861],{},[442,14862,14863,14867],{},[423,14864,14865],{},[318,14866,449],{"mathvariant":448},[312,14868,14869,14871,14873],{},[318,14870,454],{},[423,14872,457],{},[315,14874,14875,14877],{},[318,14876,320],{},[318,14878,323],{},[312,14880,14881,14883,14897,14903,14905,14907,14913,14915,14921,14923,14929,14931,15015],{},[423,14882,466],{"fence":333},[1702,14884,14885,14887,14895],{},[423,14886,2704],{},[312,14888,14889,14891,14893],{},[318,14890,1711],{},[423,14892,438],{},[778,14894,1425],{},[318,14896,1454],{},[315,14898,14899,14901],{},[318,14900,803],{"mathvariant":802},[318,14902,323],{},[318,14904,2289],{},[423,14906,2292],{},[315,14908,14909,14911],{},[318,14910,320],{},[318,14912,323],{},[423,14914,426],{"stretchy":425},[315,14916,14917,14919],{},[318,14918,216],{},[318,14920,1711],{},[318,14922,820],{"mathvariant":802},[315,14924,14925,14927],{},[318,14926,1422],{},[318,14928,1711],{},[423,14930,435],{"stretchy":425},[312,14932,14933,14935,14953,14955,14957,14967,14969,14979,14981,14997,14999,15001,15003,15005,15011,15013],{},[423,14934,426],{"fence":333},[1702,14936,14937,14939,14951],{},[423,14938,2704],{},[312,14940,14941,14947,14949],{},[12712,14942,14943,14945],{},[318,14944,1711],{},[423,14946,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,14948,438],{},[318,14950,1711],{},[318,14952,1454],{},[318,14954,469],{},[423,14956,426],{"stretchy":425},[315,14958,14959,14961],{},[318,14960,1422],{},[12712,14962,14963,14965],{},[318,14964,1711],{},[423,14966,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,14968,1428],{"separator":333},[315,14970,14971,14973],{},[318,14972,216],{},[12712,14974,14975,14977],{},[318,14976,1711],{},[423,14978,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,14980,1428],{"separator":333},[315,14982,14983,14985],{},[318,14984,1422],{},[312,14986,14987,14993,14995],{},[12712,14988,14989,14991],{},[318,14990,1711],{},[423,14992,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,14994,776],{},[778,14996,780],{},[423,14998,435],{"stretchy":425},[423,15000,9426],{},[318,15002,14297],{},[423,15004,426],{"stretchy":425},[315,15006,15007,15009],{},[318,15008,1422],{},[318,15010,1711],{},[423,15012,435],{"stretchy":425},[423,15014,435],{"fence":333},[423,15016,478],{"fence":333},[325,15018,15019],{"encoding":327},"\\nabla_{\\theta} J(\\pi_{\\theta}) = \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) \\left(\\sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1}) - b(s_t)\\right)\\right]",[296,15021,15023,15133],{"className":15022,"ariaHidden":333},[332],[296,15024,15026,15029,15072,15075,15078,15121,15124,15127,15130],{"className":15025},[337],[296,15027],{"className":15028,"style":491},[341],[296,15030,15032,15035],{"className":15031},[346],[296,15033,803],{"className":15034},[346],[296,15036,15038],{"className":15037},[355],[296,15039,15041,15064],{"className":15040},[359,360],[296,15042,15044,15061],{"className":15043},[364],[296,15045,15047],{"className":15046,"style":369},[368],[296,15048,15049,15052],{"style":1005},[296,15050],{"className":15051,"style":377},[376],[296,15053,15055],{"className":15054},[381,382,383,384],[296,15056,15058],{"className":15057},[346,384],[296,15059,323],{"className":15060,"style":391},[346,350,384],[296,15062,396],{"className":15063},[395],[296,15065,15067],{"className":15066},[364],[296,15068,15070],{"className":15069,"style":403},[368],[296,15071],{},[296,15073,421],{"className":15074,"style":495},[346,350],[296,15076,426],{"className":15077},[499],[296,15079,15081,15084],{"className":15080},[346],[296,15082,320],{"className":15083,"style":351},[346,350],[296,15085,15087],{"className":15086},[355],[296,15088,15090,15113],{"className":15089},[359,360],[296,15091,15093,15110],{"className":15092},[364],[296,15094,15096],{"className":15095,"style":369},[368],[296,15097,15098,15101],{"style":372},[296,15099],{"className":15100,"style":377},[376],[296,15102,15104],{"className":15103},[381,382,383,384],[296,15105,15107],{"className":15106},[346,384],[296,15108,323],{"className":15109,"style":391},[346,350,384],[296,15111,396],{"className":15112},[395],[296,15114,15116],{"className":15115},[364],[296,15117,15119],{"className":15118,"style":403},[368],[296,15120],{},[296,15122,435],{"className":15123},[546],[296,15125],{"className":15126,"style":551},[550],[296,15128,438],{"className":15129},[555],[296,15131],{"className":15132,"style":551},[550],[296,15134,15136,15139,15235,15238],{"className":15135},[337],[296,15137],{"className":15138,"style":12895},[341],[296,15140,15142],{"className":15141},[346],[296,15143,15145],{"className":15144},[572,573],[296,15146,15148,15227],{"className":15147},[359,360],[296,15149,15151,15224],{"className":15150},[364],[296,15152,15154,15214],{"className":15153,"style":583},[368],[296,15155,15156,15159],{"style":586},[296,15157],{"className":15158,"style":590},[376],[296,15160,15162],{"className":15161},[381,382,383,384],[296,15163,15165,15168,15171],{"className":15164},[346,384],[296,15166,454],{"className":15167,"style":600},[346,350,384],[296,15169,457],{"className":15170},[555,384],[296,15172,15174,15177],{"className":15173},[346,384],[296,15175,320],{"className":15176,"style":351},[346,350,384],[296,15178,15180],{"className":15179},[355],[296,15181,15183,15206],{"className":15182},[359,360],[296,15184,15186,15203],{"className":15185},[364],[296,15187,15189],{"className":15188,"style":622},[368],[296,15190,15191,15194],{"style":625},[296,15192],{"className":15193,"style":629},[376],[296,15195,15197],{"className":15196},[381,633,634,384],[296,15198,15200],{"className":15199},[346,384],[296,15201,323],{"className":15202,"style":391},[346,350,384],[296,15204,396],{"className":15205},[395],[296,15207,15209],{"className":15208},[364],[296,15210,15212],{"className":15211,"style":650},[368],[296,15213],{},[296,15215,15216,15219],{"style":655},[296,15217],{"className":15218,"style":590},[376],[296,15220,15221],{},[296,15222,449],{"className":15223},[572,664],[296,15225,396],{"className":15226},[395],[296,15228,15230],{"className":15229},[364],[296,15231,15233],{"className":15232,"style":674},[368],[296,15234],{},[296,15236],{"className":15237,"style":973},[550],[296,15239,15241,15247,15314,15317,15360,15363,15368,15371,15414,15417,15457,15460,15500,15503,15506,15918],{"className":15240},[977],[296,15242,15244],{"className":15243,"style":1085},[499,1084],[296,15245,466],{"className":15246},[3026,7043],[296,15248,15250],{"className":15249},[572,573],[296,15251,15253,15306],{"className":15252},[359,360],[296,15254,15256,15303],{"className":15255},[364],[296,15257,15259,15279,15289],{"className":15258,"style":1923},[368],[296,15260,15261,15264],{"style":1926},[296,15262],{"className":15263,"style":1930},[376],[296,15265,15267],{"className":15266},[381,382,383,384],[296,15268,15270,15273,15276],{"className":15269},[346,384],[296,15271,1711],{"className":15272},[346,350,384],[296,15274,438],{"className":15275},[555,384],[296,15277,1425],{"className":15278},[346,384],[296,15280,15281,15284],{"style":1948},[296,15282],{"className":15283,"style":1930},[376],[296,15285,15286],{},[296,15287,2704],{"className":15288},[572,1957,1958],[296,15290,15291,15294],{"style":1961},[296,15292],{"className":15293,"style":1930},[376],[296,15295,15297],{"className":15296},[381,382,383,384],[296,15298,15300],{"className":15299},[346,384],[296,15301,1454],{"className":15302,"style":1632},[346,350,384],[296,15304,396],{"className":15305},[395],[296,15307,15309],{"className":15308},[364],[296,15310,15312],{"className":15311,"style":1983},[368],[296,15313],{},[296,15315],{"className":15316,"style":973},[550],[296,15318,15320,15323],{"className":15319},[346],[296,15321,803],{"className":15322},[346],[296,15324,15326],{"className":15325},[355],[296,15327,15329,15352],{"className":15328},[359,360],[296,15330,15332,15349],{"className":15331},[364],[296,15333,15335],{"className":15334,"style":369},[368],[296,15336,15337,15340],{"style":1005},[296,15338],{"className":15339,"style":377},[376],[296,15341,15343],{"className":15342},[381,382,383,384],[296,15344,15346],{"className":15345},[346,384],[296,15347,323],{"className":15348,"style":391},[346,350,384],[296,15350,396],{"className":15351},[395],[296,15353,15355],{"className":15354},[364],[296,15356,15358],{"className":15357,"style":403},[368],[296,15359],{},[296,15361],{"className":15362,"style":973},[550],[296,15364,2312,15366],{"className":15365},[572],[296,15367,2316],{"style":2315},[296,15369],{"className":15370,"style":973},[550],[296,15372,15374,15377],{"className":15373},[346],[296,15375,320],{"className":15376,"style":351},[346,350],[296,15378,15380],{"className":15379},[355],[296,15381,15383,15406],{"className":15382},[359,360],[296,15384,15386,15403],{"className":15385},[364],[296,15387,15389],{"className":15388,"style":369},[368],[296,15390,15391,15394],{"style":372},[296,15392],{"className":15393,"style":377},[376],[296,15395,15397],{"className":15396},[381,382,383,384],[296,15398,15400],{"className":15399},[346,384],[296,15401,323],{"className":15402,"style":391},[346,350,384],[296,15404,396],{"className":15405},[395],[296,15407,15409],{"className":15408},[364],[296,15410,15412],{"className":15411,"style":403},[368],[296,15413],{},[296,15415,426],{"className":15416},[499],[296,15418,15420,15423],{"className":15419},[346],[296,15421,216],{"className":15422},[346,350],[296,15424,15426],{"className":15425},[355],[296,15427,15429,15449],{"className":15428},[359,360],[296,15430,15432,15446],{"className":15431},[364],[296,15433,15435],{"className":15434,"style":2065},[368],[296,15436,15437,15440],{"style":1005},[296,15438],{"className":15439,"style":377},[376],[296,15441,15443],{"className":15442},[381,382,383,384],[296,15444,1711],{"className":15445},[346,350,384],[296,15447,396],{"className":15448},[395],[296,15450,15452],{"className":15451},[364],[296,15453,15455],{"className":15454,"style":403},[368],[296,15456],{},[296,15458,820],{"className":15459},[346],[296,15461,15463,15466],{"className":15462},[346],[296,15464,1422],{"className":15465},[346,350],[296,15467,15469],{"className":15468},[355],[296,15470,15472,15492],{"className":15471},[359,360],[296,15473,15475,15489],{"className":15474},[364],[296,15476,15478],{"className":15477,"style":2065},[368],[296,15479,15480,15483],{"style":1005},[296,15481],{"className":15482,"style":377},[376],[296,15484,15486],{"className":15485},[381,382,383,384],[296,15487,1711],{"className":15488},[346,350,384],[296,15490,396],{"className":15491},[395],[296,15493,15495],{"className":15494},[364],[296,15496,15498],{"className":15497,"style":403},[368],[296,15499],{},[296,15501,435],{"className":15502},[546],[296,15504],{"className":15505,"style":973},[550],[296,15507,15509,15515,15608,15611,15614,15617,15689,15692,15695,15767,15770,15773,15851,15854,15857,15860,15863,15866,15869,15909,15912],{"className":15508},[977],[296,15510,15512],{"className":15511,"style":1085},[499,1084],[296,15513,426],{"className":15514},[3026,7043],[296,15516,15518],{"className":15517},[572,573],[296,15519,15521,15600],{"className":15520},[359,360],[296,15522,15524,15597],{"className":15523},[364],[296,15525,15527,15576,15586],{"className":15526,"style":1923},[368],[296,15528,15529,15532],{"style":13277},[296,15530],{"className":15531,"style":1930},[376],[296,15533,15535],{"className":15534},[381,382,383,384],[296,15536,15538,15570,15573],{"className":15537},[346,384],[296,15539,15541,15544],{"className":15540},[346,384],[296,15542,1711],{"className":15543},[346,350,384],[296,15545,15547],{"className":15546},[355],[296,15548,15550],{"className":15549},[359],[296,15551,15553],{"className":15552},[364],[296,15554,15556],{"className":15555,"style":13305},[368],[296,15557,15558,15561],{"style":13308},[296,15559],{"className":15560,"style":629},[376],[296,15562,15564],{"className":15563},[381,633,634,384],[296,15565,15567],{"className":15566},[346,384],[296,15568,12718],{"className":15569},[346,384],[296,15571,438],{"className":15572},[555,384],[296,15574,1711],{"className":15575},[346,350,384],[296,15577,15578,15581],{"style":1948},[296,15579],{"className":15580,"style":1930},[376],[296,15582,15583],{},[296,15584,2704],{"className":15585},[572,1957,1958],[296,15587,15588,15591],{"style":1961},[296,15589],{"className":15590,"style":1930},[376],[296,15592,15594],{"className":15593},[381,382,383,384],[296,15595,1454],{"className":15596,"style":1632},[346,350,384],[296,15598,396],{"className":15599},[395],[296,15601,15603],{"className":15602},[364],[296,15604,15606],{"className":15605,"style":13357},[368],[296,15607],{},[296,15609],{"className":15610,"style":973},[550],[296,15612,469],{"className":15613,"style":683},[346,350],[296,15615,426],{"className":15616},[499],[296,15618,15620,15623],{"className":15619},[346],[296,15621,1422],{"className":15622},[346,350],[296,15624,15626],{"className":15625},[355],[296,15627,15629,15681],{"className":15628},[359,360],[296,15630,15632,15678],{"className":15631},[364],[296,15633,15635],{"className":15634,"style":13387},[368],[296,15636,15637,15640],{"style":1005},[296,15638],{"className":15639,"style":377},[376],[296,15641,15643],{"className":15642},[381,382,383,384],[296,15644,15646],{"className":15645},[346,384],[296,15647,15649,15652],{"className":15648},[346,384],[296,15650,1711],{"className":15651},[346,350,384],[296,15653,15655],{"className":15654},[355],[296,15656,15658],{"className":15657},[359],[296,15659,15661],{"className":15660},[364],[296,15662,15664],{"className":15663,"style":13305},[368],[296,15665,15666,15669],{"style":13308},[296,15667],{"className":15668,"style":629},[376],[296,15670,15672],{"className":15671},[381,633,634,384],[296,15673,15675],{"className":15674},[346,384],[296,15676,12718],{"className":15677},[346,384],[296,15679,396],{"className":15680},[395],[296,15682,15684],{"className":15683},[364],[296,15685,15687],{"className":15686,"style":403},[368],[296,15688],{},[296,15690,1428],{"className":15691},[1539],[296,15693],{"className":15694,"style":973},[550],[296,15696,15698,15701],{"className":15697},[346],[296,15699,216],{"className":15700},[346,350],[296,15702,15704],{"className":15703},[355],[296,15705,15707,15759],{"className":15706},[359,360],[296,15708,15710,15756],{"className":15709},[364],[296,15711,15713],{"className":15712,"style":13387},[368],[296,15714,15715,15718],{"style":1005},[296,15716],{"className":15717,"style":377},[376],[296,15719,15721],{"className":15720},[381,382,383,384],[296,15722,15724],{"className":15723},[346,384],[296,15725,15727,15730],{"className":15726},[346,384],[296,15728,1711],{"className":15729},[346,350,384],[296,15731,15733],{"className":15732},[355],[296,15734,15736],{"className":15735},[359],[296,15737,15739],{"className":15738},[364],[296,15740,15742],{"className":15741,"style":13305},[368],[296,15743,15744,15747],{"style":13308},[296,15745],{"className":15746,"style":629},[376],[296,15748,15750],{"className":15749},[381,633,634,384],[296,15751,15753],{"className":15752},[346,384],[296,15754,12718],{"className":15755},[346,384],[296,15757,396],{"className":15758},[395],[296,15760,15762],{"className":15761},[364],[296,15763,15765],{"className":15764,"style":403},[368],[296,15766],{},[296,15768,1428],{"className":15769},[1539],[296,15771],{"className":15772,"style":973},[550],[296,15774,15776,15779],{"className":15775},[346],[296,15777,1422],{"className":15778},[346,350],[296,15780,15782],{"className":15781},[355],[296,15783,15785,15843],{"className":15784},[359,360],[296,15786,15788,15840],{"className":15787},[364],[296,15789,15791],{"className":15790,"style":13387},[368],[296,15792,15793,15796],{"style":1005},[296,15794],{"className":15795,"style":377},[376],[296,15797,15799],{"className":15798},[381,382,383,384],[296,15800,15802,15834,15837],{"className":15801},[346,384],[296,15803,15805,15808],{"className":15804},[346,384],[296,15806,1711],{"className":15807},[346,350,384],[296,15809,15811],{"className":15810},[355],[296,15812,15814],{"className":15813},[359],[296,15815,15817],{"className":15816},[364],[296,15818,15820],{"className":15819,"style":13305},[368],[296,15821,15822,15825],{"style":13308},[296,15823],{"className":15824,"style":629},[376],[296,15826,15828],{"className":15827},[381,633,634,384],[296,15829,15831],{"className":15830},[346,384],[296,15832,12718],{"className":15833},[346,384],[296,15835,776],{"className":15836},[877,384],[296,15838,780],{"className":15839},[346,384],[296,15841,396],{"className":15842},[395],[296,15844,15846],{"className":15845},[364],[296,15847,15849],{"className":15848,"style":890},[368],[296,15850],{},[296,15852,435],{"className":15853},[546],[296,15855],{"className":15856,"style":952},[550],[296,15858,9426],{"className":15859},[877],[296,15861],{"className":15862,"style":952},[550],[296,15864,14297],{"className":15865},[346,350],[296,15867,426],{"className":15868},[499],[296,15870,15872,15875],{"className":15871},[346],[296,15873,1422],{"className":15874},[346,350],[296,15876,15878],{"className":15877},[355],[296,15879,15881,15901],{"className":15880},[359,360],[296,15882,15884,15898],{"className":15883},[364],[296,15885,15887],{"className":15886,"style":2065},[368],[296,15888,15889,15892],{"style":1005},[296,15890],{"className":15891,"style":377},[376],[296,15893,15895],{"className":15894},[381,382,383,384],[296,15896,1711],{"className":15897},[346,350,384],[296,15899,396],{"className":15900},[395],[296,15902,15904],{"className":15903},[364],[296,15905,15907],{"className":15906,"style":403},[368],[296,15908],{},[296,15910,435],{"className":15911},[546],[296,15913,15915],{"className":15914,"style":1085},[546,1084],[296,15916,435],{"className":15917},[3026,7043],[296,15919,15921],{"className":15920,"style":1085},[546,1084],[296,15922,478],{"className":15923},[3026,7043],[212,15925,15926,15927,15955,15956,8685],{},"这样使用的任何函数 ",[296,15928,15930,15943],{"className":15929},[299],[296,15931,15933],{"className":15932},[303],[305,15934,15935],{"xmlns":307},[309,15936,15937,15941],{},[312,15938,15939],{},[318,15940,14297],{},[325,15942,14297],{"encoding":327},[296,15944,15946],{"className":15945,"ariaHidden":333},[332],[296,15947,15949,15952],{"className":15948},[337],[296,15950],{"className":15951,"style":3379},[341],[296,15953,14297],{"className":15954},[346,350]," 都叫作",[247,15957,15958],{},"基线（baseline）",[212,15960,15961,15962,15966,15967,16088,16089,16159,16160,16189],{},"最常见的基线选择是",[216,15963,15965],{"href":15964},"\u002Fposts\u002Fspinning-up-rl-intro#%E4%BB%B7%E5%80%BC%E5%87%BD%E6%95%B0","同策略价值函数"," ",[296,15968,15970,15999],{"className":15969},[299],[296,15971,15973],{"className":15972},[303],[305,15974,15975],{"xmlns":307},[309,15976,15977,15996],{},[312,15978,15979,15986,15988,15994],{},[12712,15980,15981,15984],{},[318,15982,15983],{},"V",[318,15985,320],{},[423,15987,426],{"stretchy":425},[315,15989,15990,15992],{},[318,15991,1422],{},[318,15993,1711],{},[423,15995,435],{"stretchy":425},[325,15997,15998],{"encoding":327},"V^{\\pi}(s_t)",[296,16000,16002],{"className":16001,"ariaHidden":333},[332],[296,16003,16005,16008,16042,16045,16085],{"className":16004},[337],[296,16006],{"className":16007,"style":491},[341],[296,16009,16011,16014],{"className":16010},[346],[296,16012,15983],{"className":16013,"style":952},[346,350],[296,16015,16017],{"className":16016},[355],[296,16018,16020],{"className":16019},[359],[296,16021,16023],{"className":16022},[364],[296,16024,16027],{"className":16025,"style":16026},[368],"height:0.6644em;",[296,16028,16030,16033],{"style":16029},"top:-3.063em;margin-right:0.05em;",[296,16031],{"className":16032,"style":377},[376],[296,16034,16036],{"className":16035},[381,382,383,384],[296,16037,16039],{"className":16038},[346,384],[296,16040,320],{"className":16041,"style":351},[346,350,384],[296,16043,426],{"className":16044},[499],[296,16046,16048,16051],{"className":16047},[346],[296,16049,1422],{"className":16050},[346,350],[296,16052,16054],{"className":16053},[355],[296,16055,16057,16077],{"className":16056},[359,360],[296,16058,16060,16074],{"className":16059},[364],[296,16061,16063],{"className":16062,"style":2065},[368],[296,16064,16065,16068],{"style":1005},[296,16066],{"className":16067,"style":377},[376],[296,16069,16071],{"className":16070},[381,382,383,384],[296,16072,1711],{"className":16073},[346,350,384],[296,16075,396],{"className":16076},[395],[296,16078,16080],{"className":16079},[364],[296,16081,16083],{"className":16082,"style":403},[368],[296,16084],{},[296,16086,435],{"className":16087},[546],"。回忆一下：它指智能体从状态 ",[296,16090,16092,16110],{"className":16091},[299],[296,16093,16095],{"className":16094},[303],[305,16096,16097],{"xmlns":307},[309,16098,16099,16107],{},[312,16100,16101],{},[315,16102,16103,16105],{},[318,16104,1422],{},[318,16106,1711],{},[325,16108,16109],{"encoding":327},"s_t",[296,16111,16113],{"className":16112,"ariaHidden":333},[332],[296,16114,16116,16119],{"className":16115},[337],[296,16117],{"className":16118,"style":342},[341],[296,16120,16122,16125],{"className":16121},[346],[296,16123,1422],{"className":16124},[346,350],[296,16126,16128],{"className":16127},[355],[296,16129,16131,16151],{"className":16130},[359,360],[296,16132,16134,16148],{"className":16133},[364],[296,16135,16137],{"className":16136,"style":2065},[368],[296,16138,16139,16142],{"style":1005},[296,16140],{"className":16141,"style":377},[376],[296,16143,16145],{"className":16144},[381,382,383,384],[296,16146,1711],{"className":16147},[346,350,384],[296,16149,396],{"className":16150},[395],[296,16152,16154],{"className":16153},[364],[296,16155,16157],{"className":16156,"style":403},[368],[296,16158],{}," 出发、之后一直按策略 ",[296,16161,16163,16177],{"className":16162},[299],[296,16164,16166],{"className":16165},[303],[305,16167,16168],{"xmlns":307},[309,16169,16170,16174],{},[312,16171,16172],{},[318,16173,320],{},[325,16175,16176],{"encoding":327},"\\pi",[296,16178,16180],{"className":16179,"ariaHidden":333},[332],[296,16181,16183,16186],{"className":16182},[337],[296,16184],{"className":16185,"style":1473},[341],[296,16187,320],{"className":16188,"style":351},[346,350]," 行动所能得到的平均回报。",[212,16191,16192,16193,16389],{},"经验上，取 ",[296,16194,16196,16238],{"className":16195},[299],[296,16197,16199],{"className":16198},[303],[305,16200,16201],{"xmlns":307},[309,16202,16203,16235],{},[312,16204,16205,16207,16209,16215,16217,16219,16225,16227,16233],{},[318,16206,14297],{},[423,16208,426],{"stretchy":425},[315,16210,16211,16213],{},[318,16212,1422],{},[318,16214,1711],{},[423,16216,435],{"stretchy":425},[423,16218,438],{},[12712,16220,16221,16223],{},[318,16222,15983],{},[318,16224,320],{},[423,16226,426],{"stretchy":425},[315,16228,16229,16231],{},[318,16230,1422],{},[318,16232,1711],{},[423,16234,435],{"stretchy":425},[325,16236,16237],{"encoding":327},"b(s_t) = V^{\\pi}(s_t)",[296,16239,16241,16305],{"className":16240,"ariaHidden":333},[332],[296,16242,16244,16247,16250,16253,16293,16296,16299,16302],{"className":16243},[337],[296,16245],{"className":16246,"style":491},[341],[296,16248,14297],{"className":16249},[346,350],[296,16251,426],{"className":16252},[499],[296,16254,16256,16259],{"className":16255},[346],[296,16257,1422],{"className":16258},[346,350],[296,16260,16262],{"className":16261},[355],[296,16263,16265,16285],{"className":16264},[359,360],[296,16266,16268,16282],{"className":16267},[364],[296,16269,16271],{"className":16270,"style":2065},[368],[296,16272,16273,16276],{"style":1005},[296,16274],{"className":16275,"style":377},[376],[296,16277,16279],{"className":16278},[381,382,383,384],[296,16280,1711],{"className":16281},[346,350,384],[296,16283,396],{"className":16284},[395],[296,16286,16288],{"className":16287},[364],[296,16289,16291],{"className":16290,"style":403},[368],[296,16292],{},[296,16294,435],{"className":16295},[546],[296,16297],{"className":16298,"style":551},[550],[296,16300,438],{"className":16301},[555],[296,16303],{"className":16304,"style":551},[550],[296,16306,16308,16311,16343,16346,16386],{"className":16307},[337],[296,16309],{"className":16310,"style":491},[341],[296,16312,16314,16317],{"className":16313},[346],[296,16315,15983],{"className":16316,"style":952},[346,350],[296,16318,16320],{"className":16319},[355],[296,16321,16323],{"className":16322},[359],[296,16324,16326],{"className":16325},[364],[296,16327,16329],{"className":16328,"style":16026},[368],[296,16330,16331,16334],{"style":16029},[296,16332],{"className":16333,"style":377},[376],[296,16335,16337],{"className":16336},[381,382,383,384],[296,16338,16340],{"className":16339},[346,384],[296,16341,320],{"className":16342,"style":351},[346,350,384],[296,16344,426],{"className":16345},[499],[296,16347,16349,16352],{"className":16348},[346],[296,16350,1422],{"className":16351},[346,350],[296,16353,16355],{"className":16354},[355],[296,16356,16358,16378],{"className":16357},[359,360],[296,16359,16361,16375],{"className":16360},[364],[296,16362,16364],{"className":16363,"style":2065},[368],[296,16365,16366,16369],{"style":1005},[296,16367],{"className":16368,"style":377},[376],[296,16370,16372],{"className":16371},[381,382,383,384],[296,16373,1711],{"className":16374},[346,350,384],[296,16376,396],{"className":16377},[395],[296,16379,16381],{"className":16380},[364],[296,16382,16384],{"className":16383,"style":403},[368],[296,16385],{},[296,16387,435],{"className":16388},[546]," 能切实降低策略梯度样本估计的方差，策略学得更快也更稳。从概念上讲，这个选择也有吸引力：它编码了这样一个直觉，如果智能体得到的就是它预期的，它对此应该「无感」。",[208,16391,16392,16645,16723,17407],{"title":8676},[212,16393,16394,16395,16512,16513,16644],{},"实践中 ",[296,16396,16398,16425],{"className":16397},[299],[296,16399,16401],{"className":16400},[303],[305,16402,16403],{"xmlns":307},[309,16404,16405,16423],{},[312,16406,16407,16413,16415,16421],{},[12712,16408,16409,16411],{},[318,16410,15983],{},[318,16412,320],{},[423,16414,426],{"stretchy":425},[315,16416,16417,16419],{},[318,16418,1422],{},[318,16420,1711],{},[423,16422,435],{"stretchy":425},[325,16424,15998],{"encoding":327},[296,16426,16428],{"className":16427,"ariaHidden":333},[332],[296,16429,16431,16434,16466,16469,16509],{"className":16430},[337],[296,16432],{"className":16433,"style":491},[341],[296,16435,16437,16440],{"className":16436},[346],[296,16438,15983],{"className":16439,"style":952},[346,350],[296,16441,16443],{"className":16442},[355],[296,16444,16446],{"className":16445},[359],[296,16447,16449],{"className":16448},[364],[296,16450,16452],{"className":16451,"style":16026},[368],[296,16453,16454,16457],{"style":16029},[296,16455],{"className":16456,"style":377},[376],[296,16458,16460],{"className":16459},[381,382,383,384],[296,16461,16463],{"className":16462},[346,384],[296,16464,320],{"className":16465,"style":351},[346,350,384],[296,16467,426],{"className":16468},[499],[296,16470,16472,16475],{"className":16471},[346],[296,16473,1422],{"className":16474},[346,350],[296,16476,16478],{"className":16477},[355],[296,16479,16481,16501],{"className":16480},[359,360],[296,16482,16484,16498],{"className":16483},[364],[296,16485,16487],{"className":16486,"style":2065},[368],[296,16488,16489,16492],{"style":1005},[296,16490],{"className":16491,"style":377},[376],[296,16493,16495],{"className":16494},[381,382,383,384],[296,16496,1711],{"className":16497},[346,350,384],[296,16499,396],{"className":16500},[395],[296,16502,16504],{"className":16503},[364],[296,16505,16507],{"className":16506,"style":403},[368],[296,16508],{},[296,16510,435],{"className":16511},[546]," 算不出来，只能近似。通常用一个神经网络 ",[296,16514,16516,16545],{"className":16515},[299],[296,16517,16519],{"className":16518},[303],[305,16520,16521],{"xmlns":307},[309,16522,16523,16542],{},[312,16524,16525,16532,16534,16540],{},[315,16526,16527,16529],{},[318,16528,15983],{},[318,16530,16531],{},"ϕ",[423,16533,426],{"stretchy":425},[315,16535,16536,16538],{},[318,16537,1422],{},[318,16539,1711],{},[423,16541,435],{"stretchy":425},[325,16543,16544],{"encoding":327},"V_{\\phi}(s_t)",[296,16546,16548],{"className":16547,"ariaHidden":333},[332],[296,16549,16551,16554,16598,16601,16641],{"className":16550},[337],[296,16552],{"className":16553,"style":7620},[341],[296,16555,16557,16560],{"className":16556},[346],[296,16558,15983],{"className":16559,"style":952},[346,350],[296,16561,16563],{"className":16562},[355],[296,16564,16566,16590],{"className":16565},[359,360],[296,16567,16569,16587],{"className":16568},[364],[296,16570,16572],{"className":16571,"style":369},[368],[296,16573,16575,16578],{"style":16574},"top:-2.55em;margin-left:-0.2222em;margin-right:0.05em;",[296,16576],{"className":16577,"style":377},[376],[296,16579,16581],{"className":16580},[381,382,383,384],[296,16582,16584],{"className":16583},[346,384],[296,16585,16531],{"className":16586},[346,350,384],[296,16588,396],{"className":16589},[395],[296,16591,16593],{"className":16592},[364],[296,16594,16596],{"className":16595,"style":7726},[368],[296,16597],{},[296,16599,426],{"className":16600},[499],[296,16602,16604,16607],{"className":16603},[346],[296,16605,1422],{"className":16606},[346,350],[296,16608,16610],{"className":16609},[355],[296,16611,16613,16633],{"className":16612},[359,360],[296,16614,16616,16630],{"className":16615},[364],[296,16617,16619],{"className":16618,"style":2065},[368],[296,16620,16621,16624],{"style":1005},[296,16622],{"className":16623,"style":377},[376],[296,16625,16627],{"className":16626},[381,382,383,384],[296,16628,1711],{"className":16629},[346,350,384],[296,16631,396],{"className":16632},[395],[296,16634,16636],{"className":16635},[364],[296,16637,16639],{"className":16638,"style":403},[368],[296,16640],{},[296,16642,435],{"className":16643},[546],"，与策略同时更新（让价值网络始终近似最新策略的价值函数）。",[212,16646,16647,16648,16722],{},"学习 ",[296,16649,16651,16669],{"className":16650},[299],[296,16652,16654],{"className":16653},[303],[305,16655,16656],{"xmlns":307},[309,16657,16658,16666],{},[312,16659,16660],{},[315,16661,16662,16664],{},[318,16663,15983],{},[318,16665,16531],{},[325,16667,16668],{"encoding":327},"V_{\\phi}",[296,16670,16672],{"className":16671,"ariaHidden":333},[332],[296,16673,16675,16679],{"className":16674},[337],[296,16676],{"className":16677,"style":16678},[341],"height:0.9694em;vertical-align:-0.2861em;",[296,16680,16682,16685],{"className":16681},[346],[296,16683,15983],{"className":16684,"style":952},[346,350],[296,16686,16688],{"className":16687},[355],[296,16689,16691,16714],{"className":16690},[359,360],[296,16692,16694,16711],{"className":16693},[364],[296,16695,16697],{"className":16696,"style":369},[368],[296,16698,16699,16702],{"style":16574},[296,16700],{"className":16701,"style":377},[376],[296,16703,16705],{"className":16704},[381,382,383,384],[296,16706,16708],{"className":16707},[346,384],[296,16709,16531],{"className":16710},[346,350,384],[296,16712,396],{"className":16713},[395],[296,16715,16717],{"className":16716},[364],[296,16718,16720],{"className":16719,"style":7726},[368],[296,16721],{}," 最简单的方法（VPG、TRPO、PPO 和 A2C 的大多数实现都这么做）是最小化均方误差目标：",[296,16724,16726],{"className":16725},[751],[296,16727,16729,16846],{"className":16728},[299],[296,16730,16732],{"className":16731},[303],[305,16733,16734],{"xmlns":307,"display":760},[309,16735,16736,16843],{},[312,16737,16738,16744,16746,16749,16751,16762,16798],{},[315,16739,16740,16742],{},[318,16741,16531],{},[318,16743,773],{},[423,16745,438],{},[318,16747,16748],{},"arg",[423,16750,2292],{},[442,16752,16753,16760],{},[312,16754,16755,16758],{},[318,16756,16757],{},"min",[423,16759,2292],{},[318,16761,16531],{},[318,16763,16764],{},[442,16765,16766,16770],{},[423,16767,16768],{},[318,16769,449],{"mathvariant":448},[312,16771,16772,16778,16780,16790,16792],{},[315,16773,16774,16776],{},[318,16775,1422],{},[318,16777,1711],{},[423,16779,1428],{"separator":333},[315,16781,16782,16788],{},[7819,16783,16784,16786],{"accent":333},[318,16785,469],{},[423,16787,7825],{},[318,16789,1711],{},[423,16791,457],{},[315,16793,16794,16796],{},[318,16795,320],{},[318,16797,773],{},[312,16799,16800,16802,16841],{},[423,16801,466],{"fence":333},[12712,16803,16804,16838],{},[312,16805,16806,16808,16814,16816,16822,16824,16826,16836],{},[423,16807,426],{"fence":333},[315,16809,16810,16812],{},[318,16811,15983],{},[318,16813,16531],{},[423,16815,426],{"stretchy":425},[315,16817,16818,16820],{},[318,16819,1422],{},[318,16821,1711],{},[423,16823,435],{"stretchy":425},[423,16825,9426],{},[315,16827,16828,16834],{},[7819,16829,16830,16832],{"accent":333},[318,16831,469],{},[423,16833,7825],{},[318,16835,1711],{},[423,16837,435],{"fence":333},[778,16839,16840],{},"2",[423,16842,478],{"fence":333},[325,16844,16845],{"encoding":327},"\\phi_k = \\arg \\min_{\\phi} \\underset{s_t, \\hat{R}_t \\sim \\pi_k}{\\mathbb{E}}\\left[\\left( V_{\\phi}(s_t) - \\hat{R}_t \\right)^2\\right]",[296,16847,16849,16904],{"className":16848,"ariaHidden":333},[332],[296,16850,16852,16855,16895,16898,16901],{"className":16851},[337],[296,16853],{"className":16854,"style":2308},[341],[296,16856,16858,16861],{"className":16857},[346],[296,16859,16531],{"className":16860},[346,350],[296,16862,16864],{"className":16863},[355],[296,16865,16867,16887],{"className":16866},[359,360],[296,16868,16870,16884],{"className":16869},[364],[296,16871,16873],{"className":16872,"style":369},[368],[296,16874,16875,16878],{"style":1005},[296,16876],{"className":16877,"style":377},[376],[296,16879,16881],{"className":16880},[381,382,383,384],[296,16882,773],{"className":16883,"style":873},[346,350,384],[296,16885,396],{"className":16886},[395],[296,16888,16890],{"className":16889},[364],[296,16891,16893],{"className":16892,"style":403},[368],[296,16894],{},[296,16896],{"className":16897,"style":551},[550],[296,16899,438],{"className":16900},[555],[296,16902],{"className":16903,"style":551},[550],[296,16905,16907,16911,16917,16920,16970,16973,17179,17182],{"className":16906},[337],[296,16908],{"className":16909,"style":16910},[341],"height:2.5148em;vertical-align:-1.0648em;",[296,16912,16914,16915],{"className":16913},[572],"ar",[296,16916,2316],{"style":2315},[296,16918],{"className":16919,"style":973},[550],[296,16921,16923],{"className":16922},[572,573],[296,16924,16926,16961],{"className":16925},[359,360],[296,16927,16929,16958],{"className":16928},[364],[296,16930,16933,16948],{"className":16931,"style":16932},[368],"height:0.6679em;",[296,16934,16936,16939],{"style":16935},"top:-2.3479em;margin-left:0em;",[296,16937],{"className":16938,"style":590},[376],[296,16940,16942],{"className":16941},[381,382,383,384],[296,16943,16945],{"className":16944},[346,384],[296,16946,16531],{"className":16947},[346,350,384],[296,16949,16950,16953],{"style":655},[296,16951],{"className":16952,"style":590},[376],[296,16954,16955],{},[296,16956,16757],{"className":16957},[572],[296,16959,396],{"className":16960},[395],[296,16962,16964],{"className":16963},[364],[296,16965,16968],{"className":16966,"style":16967},[368],"height:0.8882em;",[296,16969],{},[296,16971],{"className":16972,"style":973},[550],[296,16974,16976],{"className":16975},[346],[296,16977,16979],{"className":16978},[572,573],[296,16980,16982,17170],{"className":16981},[359,360],[296,16983,16985,17167],{"className":16984},[364],[296,16986,16988,17157],{"className":16987,"style":583},[368],[296,16989,16991,16994],{"style":16990},"top:-2.1713em;margin-left:0em;",[296,16992],{"className":16993,"style":590},[376],[296,16995,16997],{"className":16996},[381,382,383,384],[296,16998,17000,17040,17043,17114,17117],{"className":16999},[346,384],[296,17001,17003,17006],{"className":17002},[346,384],[296,17004,1422],{"className":17005},[346,350,384],[296,17007,17009],{"className":17008},[355],[296,17010,17012,17032],{"className":17011},[359,360],[296,17013,17015,17029],{"className":17014},[364],[296,17016,17018],{"className":17017,"style":14464},[368],[296,17019,17020,17023],{"style":14467},[296,17021],{"className":17022,"style":629},[376],[296,17024,17026],{"className":17025},[381,633,634,384],[296,17027,1711],{"className":17028},[346,350,384],[296,17030,396],{"className":17031},[395],[296,17033,17035],{"className":17034},[364],[296,17036,17038],{"className":17037,"style":14486},[368],[296,17039],{},[296,17041,1428],{"className":17042},[1539,384],[296,17044,17046,17079],{"className":17045},[346,384],[296,17047,17049],{"className":17048},[346,7925,384],[296,17050,17052],{"className":17051},[359],[296,17053,17055],{"className":17054},[364],[296,17056,17058,17067],{"className":17057,"style":13743},[368],[296,17059,17061,17064],{"style":17060},"top:-2.7em;",[296,17062],{"className":17063,"style":377},[376],[296,17065,469],{"className":17066,"style":683},[346,350,384],[296,17068,17070,17073],{"style":17069},"top:-2.9523em;",[296,17071],{"className":17072,"style":377},[376],[296,17074,17076],{"className":17075,"style":13761},[7951],[296,17077,7825],{"className":17078},[346,384],[296,17080,17082],{"className":17081},[355],[296,17083,17085,17106],{"className":17084},[359,360],[296,17086,17088,17103],{"className":17087},[364],[296,17089,17091],{"className":17090,"style":14464},[368],[296,17092,17094,17097],{"style":17093},"top:-2.357em;margin-left:-0.0077em;margin-right:0.0714em;",[296,17095],{"className":17096,"style":629},[376],[296,17098,17100],{"className":17099},[381,633,634,384],[296,17101,1711],{"className":17102},[346,350,384],[296,17104,396],{"className":17105},[395],[296,17107,17109],{"className":17108},[364],[296,17110,17112],{"className":17111,"style":14486},[368],[296,17113],{},[296,17115,457],{"className":17116},[555,384],[296,17118,17120,17123],{"className":17119},[346,384],[296,17121,320],{"className":17122,"style":351},[346,350,384],[296,17124,17126],{"className":17125},[355],[296,17127,17129,17149],{"className":17128},[359,360],[296,17130,17132,17146],{"className":17131},[364],[296,17133,17135],{"className":17134,"style":622},[368],[296,17136,17137,17140],{"style":625},[296,17138],{"className":17139,"style":629},[376],[296,17141,17143],{"className":17142},[381,633,634,384],[296,17144,773],{"className":17145,"style":873},[346,350,384],[296,17147,396],{"className":17148},[395],[296,17150,17152],{"className":17151},[364],[296,17153,17155],{"className":17154,"style":650},[368],[296,17156],{},[296,17158,17159,17162],{"style":655},[296,17160],{"className":17161,"style":590},[376],[296,17163,17164],{},[296,17165,449],{"className":17166},[572,664],[296,17168,396],{"className":17169},[395],[296,17171,17173],{"className":17172},[364],[296,17174,17177],{"className":17175,"style":17176},[368],"height:1.0648em;",[296,17178],{},[296,17180],{"className":17181,"style":973},[550],[296,17183,17185,17191,17401],{"className":17184},[977],[296,17186,17188],{"className":17187,"style":1085},[499,1084],[296,17189,466],{"className":17190},[3026,383],[296,17192,17194,17376],{"className":17193},[977],[296,17195,17197,17204,17247,17250,17290,17293,17296,17299,17302,17370],{"className":17196},[977],[296,17198,17200],{"className":17199,"style":1085},[499,1084],[296,17201,426],{"className":17202},[3026,17203],"size2",[296,17205,17207,17210],{"className":17206},[346],[296,17208,15983],{"className":17209,"style":952},[346,350],[296,17211,17213],{"className":17212},[355],[296,17214,17216,17239],{"className":17215},[359,360],[296,17217,17219,17236],{"className":17218},[364],[296,17220,17222],{"className":17221,"style":369},[368],[296,17223,17224,17227],{"style":16574},[296,17225],{"className":17226,"style":377},[376],[296,17228,17230],{"className":17229},[381,382,383,384],[296,17231,17233],{"className":17232},[346,384],[296,17234,16531],{"className":17235},[346,350,384],[296,17237,396],{"className":17238},[395],[296,17240,17242],{"className":17241},[364],[296,17243,17245],{"className":17244,"style":7726},[368],[296,17246],{},[296,17248,426],{"className":17249},[499],[296,17251,17253,17256],{"className":17252},[346],[296,17254,1422],{"className":17255},[346,350],[296,17257,17259],{"className":17258},[355],[296,17260,17262,17282],{"className":17261},[359,360],[296,17263,17265,17279],{"className":17264},[364],[296,17266,17268],{"className":17267,"style":2065},[368],[296,17269,17270,17273],{"style":1005},[296,17271],{"className":17272,"style":377},[376],[296,17274,17276],{"className":17275},[381,382,383,384],[296,17277,1711],{"className":17278},[346,350,384],[296,17280,396],{"className":17281},[395],[296,17283,17285],{"className":17284},[364],[296,17286,17288],{"className":17287,"style":403},[368],[296,17289],{},[296,17291,435],{"className":17292},[546],[296,17294],{"className":17295,"style":952},[550],[296,17297,9426],{"className":17298},[877],[296,17300],{"className":17301,"style":952},[550],[296,17303,17305,17336],{"className":17304},[346],[296,17306,17308],{"className":17307},[346,7925],[296,17309,17311],{"className":17310},[359],[296,17312,17314],{"className":17313},[364],[296,17315,17317,17325],{"className":17316,"style":13743},[368],[296,17318,17319,17322],{"style":655},[296,17320],{"className":17321,"style":590},[376],[296,17323,469],{"className":17324,"style":683},[346,350],[296,17326,17327,17330],{"style":13754},[296,17328],{"className":17329,"style":590},[376],[296,17331,17333],{"className":17332,"style":13761},[7951],[296,17334,7825],{"className":17335},[346],[296,17337,17339],{"className":17338},[355],[296,17340,17342,17362],{"className":17341},[359,360],[296,17343,17345,17359],{"className":17344},[364],[296,17346,17348],{"className":17347,"style":2065},[368],[296,17349,17350,17353],{"style":13779},[296,17351],{"className":17352,"style":377},[376],[296,17354,17356],{"className":17355},[381,382,383,384],[296,17357,1711],{"className":17358},[346,350,384],[296,17360,396],{"className":17361},[395],[296,17363,17365],{"className":17364},[364],[296,17366,17368],{"className":17367,"style":403},[368],[296,17369],{},[296,17371,17373],{"className":17372,"style":1085},[546,1084],[296,17374,435],{"className":17375},[3026,17203],[296,17377,17379],{"className":17378},[355],[296,17380,17382],{"className":17381},[359],[296,17383,17385],{"className":17384},[364],[296,17386,17389],{"className":17387,"style":17388},[368],"height:1.354em;",[296,17390,17392,17395],{"style":17391},"top:-3.6029em;margin-right:0.05em;",[296,17393],{"className":17394,"style":377},[376],[296,17396,17398],{"className":17397},[381,382,383,384],[296,17399,16840],{"className":17400},[346,384],[296,17402,17404],{"className":17403,"style":1085},[546,1084],[296,17405,478],{"className":17406},[3026,383],[212,17408,8391,17409,17479,17480,17508,17509,17594],{},[296,17410,17412,17430],{"className":17411},[299],[296,17413,17415],{"className":17414},[303],[305,17416,17417],{"xmlns":307},[309,17418,17419,17427],{},[312,17420,17421],{},[315,17422,17423,17425],{},[318,17424,320],{},[318,17426,773],{},[325,17428,17429],{"encoding":327},"\\pi_k",[296,17431,17433],{"className":17432,"ariaHidden":333},[332],[296,17434,17436,17439],{"className":17435},[337],[296,17437],{"className":17438,"style":342},[341],[296,17440,17442,17445],{"className":17441},[346],[296,17443,320],{"className":17444,"style":351},[346,350],[296,17446,17448],{"className":17447},[355],[296,17449,17451,17471],{"className":17450},[359,360],[296,17452,17454,17468],{"className":17453},[364],[296,17455,17457],{"className":17456,"style":369},[368],[296,17458,17459,17462],{"style":372},[296,17460],{"className":17461,"style":377},[376],[296,17463,17465],{"className":17464},[381,382,383,384],[296,17466,773],{"className":17467,"style":873},[346,350,384],[296,17469,396],{"className":17470},[395],[296,17472,17474],{"className":17473},[364],[296,17475,17477],{"className":17476,"style":403},[368],[296,17478],{}," 是第 ",[296,17481,17483,17496],{"className":17482},[299],[296,17484,17486],{"className":17485},[303],[305,17487,17488],{"xmlns":307},[309,17489,17490,17494],{},[312,17491,17492],{},[318,17493,773],{},[325,17495,773],{"encoding":327},[296,17497,17499],{"className":17498,"ariaHidden":333},[332],[296,17500,17502,17505],{"className":17501},[337],[296,17503],{"className":17504,"style":3379},[341],[296,17506,773],{"className":17507,"style":873},[346,350]," 轮的策略。具体做法是从上一轮的价值参数 ",[296,17510,17512,17536],{"className":17511},[299],[296,17513,17515],{"className":17514},[303],[305,17516,17517],{"xmlns":307},[309,17518,17519,17533],{},[312,17520,17521],{},[315,17522,17523,17525],{},[318,17524,16531],{},[312,17526,17527,17529,17531],{},[318,17528,773],{},[423,17530,9426],{},[778,17532,780],{},[325,17534,17535],{"encoding":327},"\\phi_{k-1}",[296,17537,17539],{"className":17538,"ariaHidden":333},[332],[296,17540,17542,17545],{"className":17541},[337],[296,17543],{"className":17544,"style":839},[341],[296,17546,17548,17551],{"className":17547},[346],[296,17549,16531],{"className":17550},[346,350],[296,17552,17554],{"className":17553},[355],[296,17555,17557,17586],{"className":17556},[359,360],[296,17558,17560,17583],{"className":17559},[364],[296,17561,17563],{"className":17562,"style":369},[368],[296,17564,17565,17568],{"style":1005},[296,17566],{"className":17567,"style":377},[376],[296,17569,17571],{"className":17570},[381,382,383,384],[296,17572,17574,17577,17580],{"className":17573},[346,384],[296,17575,773],{"className":17576,"style":873},[346,350,384],[296,17578,9426],{"className":17579},[877,384],[296,17581,780],{"className":17582},[346,384],[296,17584,396],{"className":17585},[395],[296,17587,17589],{"className":17588},[364],[296,17590,17592],{"className":17591,"style":890},[368],[296,17593],{}," 出发，做一步或多步梯度下降。",[289,17596,17597],{"id":17597},"策略梯度的其他形式",[212,17599,17600],{},"到目前为止我们看到，策略梯度有如下一般形式：",[296,17602,17604],{"className":17603},[751],[296,17605,17607,17720],{"className":17606},[299],[296,17608,17610],{"className":17609},[303],[305,17611,17612],{"xmlns":307,"display":760},[309,17613,17614,17717],{},[312,17615,17616,17622,17624,17626,17632,17634,17636,17656],{},[315,17617,17618,17620],{},[318,17619,803],{"mathvariant":802},[318,17621,323],{},[318,17623,421],{},[423,17625,426],{"stretchy":425},[315,17627,17628,17630],{},[318,17629,320],{},[318,17631,323],{},[423,17633,435],{"stretchy":425},[423,17635,438],{},[318,17637,17638],{},[442,17639,17640,17644],{},[423,17641,17642],{},[318,17643,449],{"mathvariant":448},[312,17645,17646,17648,17650],{},[318,17647,454],{},[423,17649,457],{},[315,17651,17652,17654],{},[318,17653,320],{},[318,17655,323],{},[312,17657,17658,17660,17674,17680,17682,17684,17690,17692,17698,17700,17706,17708,17715],{},[423,17659,466],{"fence":333},[1702,17661,17662,17664,17672],{},[423,17663,2704],{},[312,17665,17666,17668,17670],{},[318,17667,1711],{},[423,17669,438],{},[778,17671,1425],{},[318,17673,1454],{},[315,17675,17676,17678],{},[318,17677,803],{"mathvariant":802},[318,17679,323],{},[318,17681,2289],{},[423,17683,2292],{},[315,17685,17686,17688],{},[318,17687,320],{},[318,17689,323],{},[423,17691,426],{"stretchy":425},[315,17693,17694,17696],{},[318,17695,216],{},[318,17697,1711],{},[318,17699,820],{"mathvariant":802},[315,17701,17702,17704],{},[318,17703,1422],{},[318,17705,1711],{},[423,17707,435],{"stretchy":425},[315,17709,17710,17713],{},[318,17711,17712],{"mathvariant":802},"Φ",[318,17714,1711],{},[423,17716,478],{"fence":333},[325,17718,17719],{"encoding":327},"\\nabla_{\\theta} J(\\pi_{\\theta}) = \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) \\Phi_t\\right]",[296,17721,17723,17833],{"className":17722,"ariaHidden":333},[332],[296,17724,17726,17729,17772,17775,17778,17821,17824,17827,17830],{"className":17725},[337],[296,17727],{"className":17728,"style":491},[341],[296,17730,17732,17735],{"className":17731},[346],[296,17733,803],{"className":17734},[346],[296,17736,17738],{"className":17737},[355],[296,17739,17741,17764],{"className":17740},[359,360],[296,17742,17744,17761],{"className":17743},[364],[296,17745,17747],{"className":17746,"style":369},[368],[296,17748,17749,17752],{"style":1005},[296,17750],{"className":17751,"style":377},[376],[296,17753,17755],{"className":17754},[381,382,383,384],[296,17756,17758],{"className":17757},[346,384],[296,17759,323],{"className":17760,"style":391},[346,350,384],[296,17762,396],{"className":17763},[395],[296,17765,17767],{"className":17766},[364],[296,17768,17770],{"className":17769,"style":403},[368],[296,17771],{},[296,17773,421],{"className":17774,"style":495},[346,350],[296,17776,426],{"className":17777},[499],[296,17779,17781,17784],{"className":17780},[346],[296,17782,320],{"className":17783,"style":351},[346,350],[296,17785,17787],{"className":17786},[355],[296,17788,17790,17813],{"className":17789},[359,360],[296,17791,17793,17810],{"className":17792},[364],[296,17794,17796],{"className":17795,"style":369},[368],[296,17797,17798,17801],{"style":372},[296,17799],{"className":17800,"style":377},[376],[296,17802,17804],{"className":17803},[381,382,383,384],[296,17805,17807],{"className":17806},[346,384],[296,17808,323],{"className":17809,"style":391},[346,350,384],[296,17811,396],{"className":17812},[395],[296,17814,17816],{"className":17815},[364],[296,17817,17819],{"className":17818,"style":403},[368],[296,17820],{},[296,17822,435],{"className":17823},[546],[296,17825],{"className":17826,"style":551},[550],[296,17828,438],{"className":17829},[555],[296,17831],{"className":17832,"style":551},[550],[296,17834,17836,17839,17935,17938],{"className":17835},[337],[296,17837],{"className":17838,"style":1821},[341],[296,17840,17842],{"className":17841},[346],[296,17843,17845],{"className":17844},[572,573],[296,17846,17848,17927],{"className":17847},[359,360],[296,17849,17851,17924],{"className":17850},[364],[296,17852,17854,17914],{"className":17853,"style":583},[368],[296,17855,17856,17859],{"style":586},[296,17857],{"className":17858,"style":590},[376],[296,17860,17862],{"className":17861},[381,382,383,384],[296,17863,17865,17868,17871],{"className":17864},[346,384],[296,17866,454],{"className":17867,"style":600},[346,350,384],[296,17869,457],{"className":17870},[555,384],[296,17872,17874,17877],{"className":17873},[346,384],[296,17875,320],{"className":17876,"style":351},[346,350,384],[296,17878,17880],{"className":17879},[355],[296,17881,17883,17906],{"className":17882},[359,360],[296,17884,17886,17903],{"className":17885},[364],[296,17887,17889],{"className":17888,"style":622},[368],[296,17890,17891,17894],{"style":625},[296,17892],{"className":17893,"style":629},[376],[296,17895,17897],{"className":17896},[381,633,634,384],[296,17898,17900],{"className":17899},[346,384],[296,17901,323],{"className":17902,"style":391},[346,350,384],[296,17904,396],{"className":17905},[395],[296,17907,17909],{"className":17908},[364],[296,17910,17912],{"className":17911,"style":650},[368],[296,17913],{},[296,17915,17916,17919],{"style":655},[296,17917],{"className":17918,"style":590},[376],[296,17920,17921],{},[296,17922,449],{"className":17923},[572,664],[296,17925,396],{"className":17926},[395],[296,17928,17930],{"className":17929},[364],[296,17931,17933],{"className":17932,"style":674},[368],[296,17934],{},[296,17936],{"className":17937,"style":973},[550],[296,17939,17941,17947,18014,18017,18060,18063,18068,18071,18114,18117,18157,18160,18200,18203,18243],{"className":17940},[977],[296,17942,17944],{"className":17943,"style":1085},[499,1084],[296,17945,466],{"className":17946},[3026,7043],[296,17948,17950],{"className":17949},[572,573],[296,17951,17953,18006],{"className":17952},[359,360],[296,17954,17956,18003],{"className":17955},[364],[296,17957,17959,17979,17989],{"className":17958,"style":1923},[368],[296,17960,17961,17964],{"style":1926},[296,17962],{"className":17963,"style":1930},[376],[296,17965,17967],{"className":17966},[381,382,383,384],[296,17968,17970,17973,17976],{"className":17969},[346,384],[296,17971,1711],{"className":17972},[346,350,384],[296,17974,438],{"className":17975},[555,384],[296,17977,1425],{"className":17978},[346,384],[296,17980,17981,17984],{"style":1948},[296,17982],{"className":17983,"style":1930},[376],[296,17985,17986],{},[296,17987,2704],{"className":17988},[572,1957,1958],[296,17990,17991,17994],{"style":1961},[296,17992],{"className":17993,"style":1930},[376],[296,17995,17997],{"className":17996},[381,382,383,384],[296,17998,18000],{"className":17999},[346,384],[296,18001,1454],{"className":18002,"style":1632},[346,350,384],[296,18004,396],{"className":18005},[395],[296,18007,18009],{"className":18008},[364],[296,18010,18012],{"className":18011,"style":1983},[368],[296,18013],{},[296,18015],{"className":18016,"style":973},[550],[296,18018,18020,18023],{"className":18019},[346],[296,18021,803],{"className":18022},[346],[296,18024,18026],{"className":18025},[355],[296,18027,18029,18052],{"className":18028},[359,360],[296,18030,18032,18049],{"className":18031},[364],[296,18033,18035],{"className":18034,"style":369},[368],[296,18036,18037,18040],{"style":1005},[296,18038],{"className":18039,"style":377},[376],[296,18041,18043],{"className":18042},[381,382,383,384],[296,18044,18046],{"className":18045},[346,384],[296,18047,323],{"className":18048,"style":391},[346,350,384],[296,18050,396],{"className":18051},[395],[296,18053,18055],{"className":18054},[364],[296,18056,18058],{"className":18057,"style":403},[368],[296,18059],{},[296,18061],{"className":18062,"style":973},[550],[296,18064,2312,18066],{"className":18065},[572],[296,18067,2316],{"style":2315},[296,18069],{"className":18070,"style":973},[550],[296,18072,18074,18077],{"className":18073},[346],[296,18075,320],{"className":18076,"style":351},[346,350],[296,18078,18080],{"className":18079},[355],[296,18081,18083,18106],{"className":18082},[359,360],[296,18084,18086,18103],{"className":18085},[364],[296,18087,18089],{"className":18088,"style":369},[368],[296,18090,18091,18094],{"style":372},[296,18092],{"className":18093,"style":377},[376],[296,18095,18097],{"className":18096},[381,382,383,384],[296,18098,18100],{"className":18099},[346,384],[296,18101,323],{"className":18102,"style":391},[346,350,384],[296,18104,396],{"className":18105},[395],[296,18107,18109],{"className":18108},[364],[296,18110,18112],{"className":18111,"style":403},[368],[296,18113],{},[296,18115,426],{"className":18116},[499],[296,18118,18120,18123],{"className":18119},[346],[296,18121,216],{"className":18122},[346,350],[296,18124,18126],{"className":18125},[355],[296,18127,18129,18149],{"className":18128},[359,360],[296,18130,18132,18146],{"className":18131},[364],[296,18133,18135],{"className":18134,"style":2065},[368],[296,18136,18137,18140],{"style":1005},[296,18138],{"className":18139,"style":377},[376],[296,18141,18143],{"className":18142},[381,382,383,384],[296,18144,1711],{"className":18145},[346,350,384],[296,18147,396],{"className":18148},[395],[296,18150,18152],{"className":18151},[364],[296,18153,18155],{"className":18154,"style":403},[368],[296,18156],{},[296,18158,820],{"className":18159},[346],[296,18161,18163,18166],{"className":18162},[346],[296,18164,1422],{"className":18165},[346,350],[296,18167,18169],{"className":18168},[355],[296,18170,18172,18192],{"className":18171},[359,360],[296,18173,18175,18189],{"className":18174},[364],[296,18176,18178],{"className":18177,"style":2065},[368],[296,18179,18180,18183],{"style":1005},[296,18181],{"className":18182,"style":377},[376],[296,18184,18186],{"className":18185},[381,382,383,384],[296,18187,1711],{"className":18188},[346,350,384],[296,18190,396],{"className":18191},[395],[296,18193,18195],{"className":18194},[364],[296,18196,18198],{"className":18197,"style":403},[368],[296,18199],{},[296,18201,435],{"className":18202},[546],[296,18204,18206,18209],{"className":18205},[346],[296,18207,17712],{"className":18208},[346],[296,18210,18212],{"className":18211},[355],[296,18213,18215,18235],{"className":18214},[359,360],[296,18216,18218,18232],{"className":18217},[364],[296,18219,18221],{"className":18220,"style":2065},[368],[296,18222,18223,18226],{"style":1005},[296,18224],{"className":18225,"style":377},[376],[296,18227,18229],{"className":18228},[381,382,383,384],[296,18230,1711],{"className":18231},[346,350,384],[296,18233,396],{"className":18234},[395],[296,18236,18238],{"className":18237},[364],[296,18239,18241],{"className":18240,"style":403},[368],[296,18242],{},[296,18244,18246],{"className":18245,"style":1085},[546,1084],[296,18247,478],{"className":18248},[3026,7043],[212,18250,8391,18251,18321],{},[296,18252,18254,18272],{"className":18253},[299],[296,18255,18257],{"className":18256},[303],[305,18258,18259],{"xmlns":307},[309,18260,18261,18269],{},[312,18262,18263],{},[315,18264,18265,18267],{},[318,18266,17712],{"mathvariant":802},[318,18268,1711],{},[325,18270,18271],{"encoding":327},"\\Phi_t",[296,18273,18275],{"className":18274,"ariaHidden":333},[332],[296,18276,18278,18281],{"className":18277},[337],[296,18279],{"className":18280,"style":9924},[341],[296,18282,18284,18287],{"className":18283},[346],[296,18285,17712],{"className":18286},[346],[296,18288,18290],{"className":18289},[355],[296,18291,18293,18313],{"className":18292},[359,360],[296,18294,18296,18310],{"className":18295},[364],[296,18297,18299],{"className":18298,"style":2065},[368],[296,18300,18301,18304],{"style":1005},[296,18302],{"className":18303,"style":377},[376],[296,18305,18307],{"className":18306},[381,382,383,384],[296,18308,1711],{"className":18309},[346,350,384],[296,18311,396],{"className":18312},[395],[296,18314,18316],{"className":18315},[364],[296,18317,18319],{"className":18318,"style":403},[368],[296,18320],{}," 可以取",[296,18323,18325],{"className":18324},[751],[296,18326,18328,18356],{"className":18327},[299],[296,18329,18331],{"className":18330},[303],[305,18332,18333],{"xmlns":307,"display":760},[309,18334,18335,18353],{},[312,18336,18337,18343,18345,18347,18349,18351],{},[315,18338,18339,18341],{},[318,18340,17712],{"mathvariant":802},[318,18342,1711],{},[423,18344,438],{},[318,18346,469],{},[423,18348,426],{"stretchy":425},[318,18350,454],{},[423,18352,435],{"stretchy":425},[325,18354,18355],{"encoding":327},"\\Phi_t = R(\\tau)",[296,18357,18359,18414],{"className":18358,"ariaHidden":333},[332],[296,18360,18362,18365,18405,18408,18411],{"className":18361},[337],[296,18363],{"className":18364,"style":9924},[341],[296,18366,18368,18371],{"className":18367},[346],[296,18369,17712],{"className":18370},[346],[296,18372,18374],{"className":18373},[355],[296,18375,18377,18397],{"className":18376},[359,360],[296,18378,18380,18394],{"className":18379},[364],[296,18381,18383],{"className":18382,"style":2065},[368],[296,18384,18385,18388],{"style":1005},[296,18386],{"className":18387,"style":377},[376],[296,18389,18391],{"className":18390},[381,382,383,384],[296,18392,1711],{"className":18393},[346,350,384],[296,18395,396],{"className":18396},[395],[296,18398,18400],{"className":18399},[364],[296,18401,18403],{"className":18402,"style":403},[368],[296,18404],{},[296,18406],{"className":18407,"style":551},[550],[296,18409,438],{"className":18410},[555],[296,18412],{"className":18413,"style":551},[550],[296,18415,18417,18420,18423,18426,18429],{"className":18416},[337],[296,18418],{"className":18419,"style":491},[341],[296,18421,469],{"className":18422,"style":683},[346,350],[296,18424,426],{"className":18425},[499],[296,18427,454],{"className":18428,"style":600},[346,350],[296,18430,435],{"className":18431},[546],[212,18433,18434],{},"或",[296,18436,18438],{"className":18437},[751],[296,18439,18441,18525],{"className":18440},[299],[296,18442,18444],{"className":18443},[303],[305,18445,18446],{"xmlns":307,"display":760},[309,18447,18448,18522],{},[312,18449,18450,18456,18458,18476,18478,18480,18490,18492,18502,18504,18520],{},[315,18451,18452,18454],{},[318,18453,17712],{"mathvariant":802},[318,18455,1711],{},[423,18457,438],{},[1702,18459,18460,18462,18474],{},[423,18461,2704],{},[312,18463,18464,18470,18472],{},[12712,18465,18466,18468],{},[318,18467,1711],{},[423,18469,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,18471,438],{},[318,18473,1711],{},[318,18475,1454],{},[318,18477,469],{},[423,18479,426],{"stretchy":425},[315,18481,18482,18484],{},[318,18483,1422],{},[12712,18485,18486,18488],{},[318,18487,1711],{},[423,18489,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,18491,1428],{"separator":333},[315,18493,18494,18496],{},[318,18495,216],{},[12712,18497,18498,18500],{},[318,18499,1711],{},[423,18501,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,18503,1428],{"separator":333},[315,18505,18506,18508],{},[318,18507,1422],{},[312,18509,18510,18516,18518],{},[12712,18511,18512,18514],{},[318,18513,1711],{},[423,18515,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,18517,776],{},[778,18519,780],{},[423,18521,435],{"stretchy":425},[325,18523,18524],{"encoding":327},"\\Phi_t = \\sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1})",[296,18526,18528,18583],{"className":18527,"ariaHidden":333},[332],[296,18529,18531,18534,18574,18577,18580],{"className":18530},[337],[296,18532],{"className":18533,"style":9924},[341],[296,18535,18537,18540],{"className":18536},[346],[296,18538,17712],{"className":18539},[346],[296,18541,18543],{"className":18542},[355],[296,18544,18546,18566],{"className":18545},[359,360],[296,18547,18549,18563],{"className":18548},[364],[296,18550,18552],{"className":18551,"style":2065},[368],[296,18553,18554,18557],{"style":1005},[296,18555],{"className":18556,"style":377},[376],[296,18558,18560],{"className":18559},[381,382,383,384],[296,18561,1711],{"className":18562},[346,350,384],[296,18564,396],{"className":18565},[395],[296,18567,18569],{"className":18568},[364],[296,18570,18572],{"className":18571,"style":403},[368],[296,18573],{},[296,18575],{"className":18576,"style":551},[550],[296,18578,438],{"className":18579},[555],[296,18581],{"className":18582,"style":551},[550],[296,18584,18586,18589,18682,18685,18688,18691,18763,18766,18769,18841,18844,18847,18925],{"className":18585},[337],[296,18587],{"className":18588,"style":12895},[341],[296,18590,18592],{"className":18591},[572,573],[296,18593,18595,18674],{"className":18594},[359,360],[296,18596,18598,18671],{"className":18597},[364],[296,18599,18601,18650,18660],{"className":18600,"style":1923},[368],[296,18602,18603,18606],{"style":13277},[296,18604],{"className":18605,"style":1930},[376],[296,18607,18609],{"className":18608},[381,382,383,384],[296,18610,18612,18644,18647],{"className":18611},[346,384],[296,18613,18615,18618],{"className":18614},[346,384],[296,18616,1711],{"className":18617},[346,350,384],[296,18619,18621],{"className":18620},[355],[296,18622,18624],{"className":18623},[359],[296,18625,18627],{"className":18626},[364],[296,18628,18630],{"className":18629,"style":13305},[368],[296,18631,18632,18635],{"style":13308},[296,18633],{"className":18634,"style":629},[376],[296,18636,18638],{"className":18637},[381,633,634,384],[296,18639,18641],{"className":18640},[346,384],[296,18642,12718],{"className":18643},[346,384],[296,18645,438],{"className":18646},[555,384],[296,18648,1711],{"className":18649},[346,350,384],[296,18651,18652,18655],{"style":1948},[296,18653],{"className":18654,"style":1930},[376],[296,18656,18657],{},[296,18658,2704],{"className":18659},[572,1957,1958],[296,18661,18662,18665],{"style":1961},[296,18663],{"className":18664,"style":1930},[376],[296,18666,18668],{"className":18667},[381,382,383,384],[296,18669,1454],{"className":18670,"style":1632},[346,350,384],[296,18672,396],{"className":18673},[395],[296,18675,18677],{"className":18676},[364],[296,18678,18680],{"className":18679,"style":13357},[368],[296,18681],{},[296,18683],{"className":18684,"style":973},[550],[296,18686,469],{"className":18687,"style":683},[346,350],[296,18689,426],{"className":18690},[499],[296,18692,18694,18697],{"className":18693},[346],[296,18695,1422],{"className":18696},[346,350],[296,18698,18700],{"className":18699},[355],[296,18701,18703,18755],{"className":18702},[359,360],[296,18704,18706,18752],{"className":18705},[364],[296,18707,18709],{"className":18708,"style":13387},[368],[296,18710,18711,18714],{"style":1005},[296,18712],{"className":18713,"style":377},[376],[296,18715,18717],{"className":18716},[381,382,383,384],[296,18718,18720],{"className":18719},[346,384],[296,18721,18723,18726],{"className":18722},[346,384],[296,18724,1711],{"className":18725},[346,350,384],[296,18727,18729],{"className":18728},[355],[296,18730,18732],{"className":18731},[359],[296,18733,18735],{"className":18734},[364],[296,18736,18738],{"className":18737,"style":13305},[368],[296,18739,18740,18743],{"style":13308},[296,18741],{"className":18742,"style":629},[376],[296,18744,18746],{"className":18745},[381,633,634,384],[296,18747,18749],{"className":18748},[346,384],[296,18750,12718],{"className":18751},[346,384],[296,18753,396],{"className":18754},[395],[296,18756,18758],{"className":18757},[364],[296,18759,18761],{"className":18760,"style":403},[368],[296,18762],{},[296,18764,1428],{"className":18765},[1539],[296,18767],{"className":18768,"style":973},[550],[296,18770,18772,18775],{"className":18771},[346],[296,18773,216],{"className":18774},[346,350],[296,18776,18778],{"className":18777},[355],[296,18779,18781,18833],{"className":18780},[359,360],[296,18782,18784,18830],{"className":18783},[364],[296,18785,18787],{"className":18786,"style":13387},[368],[296,18788,18789,18792],{"style":1005},[296,18790],{"className":18791,"style":377},[376],[296,18793,18795],{"className":18794},[381,382,383,384],[296,18796,18798],{"className":18797},[346,384],[296,18799,18801,18804],{"className":18800},[346,384],[296,18802,1711],{"className":18803},[346,350,384],[296,18805,18807],{"className":18806},[355],[296,18808,18810],{"className":18809},[359],[296,18811,18813],{"className":18812},[364],[296,18814,18816],{"className":18815,"style":13305},[368],[296,18817,18818,18821],{"style":13308},[296,18819],{"className":18820,"style":629},[376],[296,18822,18824],{"className":18823},[381,633,634,384],[296,18825,18827],{"className":18826},[346,384],[296,18828,12718],{"className":18829},[346,384],[296,18831,396],{"className":18832},[395],[296,18834,18836],{"className":18835},[364],[296,18837,18839],{"className":18838,"style":403},[368],[296,18840],{},[296,18842,1428],{"className":18843},[1539],[296,18845],{"className":18846,"style":973},[550],[296,18848,18850,18853],{"className":18849},[346],[296,18851,1422],{"className":18852},[346,350],[296,18854,18856],{"className":18855},[355],[296,18857,18859,18917],{"className":18858},[359,360],[296,18860,18862,18914],{"className":18861},[364],[296,18863,18865],{"className":18864,"style":13387},[368],[296,18866,18867,18870],{"style":1005},[296,18868],{"className":18869,"style":377},[376],[296,18871,18873],{"className":18872},[381,382,383,384],[296,18874,18876,18908,18911],{"className":18875},[346,384],[296,18877,18879,18882],{"className":18878},[346,384],[296,18880,1711],{"className":18881},[346,350,384],[296,18883,18885],{"className":18884},[355],[296,18886,18888],{"className":18887},[359],[296,18889,18891],{"className":18890},[364],[296,18892,18894],{"className":18893,"style":13305},[368],[296,18895,18896,18899],{"style":13308},[296,18897],{"className":18898,"style":629},[376],[296,18900,18902],{"className":18901},[381,633,634,384],[296,18903,18905],{"className":18904},[346,384],[296,18906,12718],{"className":18907},[346,384],[296,18909,776],{"className":18910},[877,384],[296,18912,780],{"className":18913},[346,384],[296,18915,396],{"className":18916},[395],[296,18918,18920],{"className":18919},[364],[296,18921,18923],{"className":18922,"style":890},[368],[296,18924],{},[296,18926,435],{"className":18927},[546],[212,18929,18434],{},[296,18931,18933],{"className":18932},[751],[296,18934,18936,19034],{"className":18935},[299],[296,18937,18939],{"className":18938},[303],[305,18940,18941],{"xmlns":307,"display":760},[309,18942,18943,19031],{},[312,18944,18945,18951,18953,18971,18973,18975,18985,18987,18997,18999,19015,19017,19019,19021,19023,19029],{},[315,18946,18947,18949],{},[318,18948,17712],{"mathvariant":802},[318,18950,1711],{},[423,18952,438],{},[1702,18954,18955,18957,18969],{},[423,18956,2704],{},[312,18958,18959,18965,18967],{},[12712,18960,18961,18963],{},[318,18962,1711],{},[423,18964,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,18966,438],{},[318,18968,1711],{},[318,18970,1454],{},[318,18972,469],{},[423,18974,426],{"stretchy":425},[315,18976,18977,18979],{},[318,18978,1422],{},[12712,18980,18981,18983],{},[318,18982,1711],{},[423,18984,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,18986,1428],{"separator":333},[315,18988,18989,18991],{},[318,18990,216],{},[12712,18992,18993,18995],{},[318,18994,1711],{},[423,18996,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,18998,1428],{"separator":333},[315,19000,19001,19003],{},[318,19002,1422],{},[312,19004,19005,19011,19013],{},[12712,19006,19007,19009],{},[318,19008,1711],{},[423,19010,12718],{"mathvariant":802,"lspace":3784,"rspace":3784},[423,19012,776],{},[778,19014,780],{},[423,19016,435],{"stretchy":425},[423,19018,9426],{},[318,19020,14297],{},[423,19022,426],{"stretchy":425},[315,19024,19025,19027],{},[318,19026,1422],{},[318,19028,1711],{},[423,19030,435],{"stretchy":425},[325,19032,19033],{"encoding":327},"\\Phi_t = \\sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1}) - b(s_t)",[296,19035,19037,19092,19446],{"className":19036,"ariaHidden":333},[332],[296,19038,19040,19043,19083,19086,19089],{"className":19039},[337],[296,19041],{"className":19042,"style":9924},[341],[296,19044,19046,19049],{"className":19045},[346],[296,19047,17712],{"className":19048},[346],[296,19050,19052],{"className":19051},[355],[296,19053,19055,19075],{"className":19054},[359,360],[296,19056,19058,19072],{"className":19057},[364],[296,19059,19061],{"className":19060,"style":2065},[368],[296,19062,19063,19066],{"style":1005},[296,19064],{"className":19065,"style":377},[376],[296,19067,19069],{"className":19068},[381,382,383,384],[296,19070,1711],{"className":19071},[346,350,384],[296,19073,396],{"className":19074},[395],[296,19076,19078],{"className":19077},[364],[296,19079,19081],{"className":19080,"style":403},[368],[296,19082],{},[296,19084],{"className":19085,"style":551},[550],[296,19087,438],{"className":19088},[555],[296,19090],{"className":19091,"style":551},[550],[296,19093,19095,19098,19191,19194,19197,19200,19272,19275,19278,19350,19353,19356,19434,19437,19440,19443],{"className":19094},[337],[296,19096],{"className":19097,"style":12895},[341],[296,19099,19101],{"className":19100},[572,573],[296,19102,19104,19183],{"className":19103},[359,360],[296,19105,19107,19180],{"className":19106},[364],[296,19108,19110,19159,19169],{"className":19109,"style":1923},[368],[296,19111,19112,19115],{"style":13277},[296,19113],{"className":19114,"style":1930},[376],[296,19116,19118],{"className":19117},[381,382,383,384],[296,19119,19121,19153,19156],{"className":19120},[346,384],[296,19122,19124,19127],{"className":19123},[346,384],[296,19125,1711],{"className":19126},[346,350,384],[296,19128,19130],{"className":19129},[355],[296,19131,19133],{"className":19132},[359],[296,19134,19136],{"className":19135},[364],[296,19137,19139],{"className":19138,"style":13305},[368],[296,19140,19141,19144],{"style":13308},[296,19142],{"className":19143,"style":629},[376],[296,19145,19147],{"className":19146},[381,633,634,384],[296,19148,19150],{"className":19149},[346,384],[296,19151,12718],{"className":19152},[346,384],[296,19154,438],{"className":19155},[555,384],[296,19157,1711],{"className":19158},[346,350,384],[296,19160,19161,19164],{"style":1948},[296,19162],{"className":19163,"style":1930},[376],[296,19165,19166],{},[296,19167,2704],{"className":19168},[572,1957,1958],[296,19170,19171,19174],{"style":1961},[296,19172],{"className":19173,"style":1930},[376],[296,19175,19177],{"className":19176},[381,382,383,384],[296,19178,1454],{"className":19179,"style":1632},[346,350,384],[296,19181,396],{"className":19182},[395],[296,19184,19186],{"className":19185},[364],[296,19187,19189],{"className":19188,"style":13357},[368],[296,19190],{},[296,19192],{"className":19193,"style":973},[550],[296,19195,469],{"className":19196,"style":683},[346,350],[296,19198,426],{"className":19199},[499],[296,19201,19203,19206],{"className":19202},[346],[296,19204,1422],{"className":19205},[346,350],[296,19207,19209],{"className":19208},[355],[296,19210,19212,19264],{"className":19211},[359,360],[296,19213,19215,19261],{"className":19214},[364],[296,19216,19218],{"className":19217,"style":13387},[368],[296,19219,19220,19223],{"style":1005},[296,19221],{"className":19222,"style":377},[376],[296,19224,19226],{"className":19225},[381,382,383,384],[296,19227,19229],{"className":19228},[346,384],[296,19230,19232,19235],{"className":19231},[346,384],[296,19233,1711],{"className":19234},[346,350,384],[296,19236,19238],{"className":19237},[355],[296,19239,19241],{"className":19240},[359],[296,19242,19244],{"className":19243},[364],[296,19245,19247],{"className":19246,"style":13305},[368],[296,19248,19249,19252],{"style":13308},[296,19250],{"className":19251,"style":629},[376],[296,19253,19255],{"className":19254},[381,633,634,384],[296,19256,19258],{"className":19257},[346,384],[296,19259,12718],{"className":19260},[346,384],[296,19262,396],{"className":19263},[395],[296,19265,19267],{"className":19266},[364],[296,19268,19270],{"className":19269,"style":403},[368],[296,19271],{},[296,19273,1428],{"className":19274},[1539],[296,19276],{"className":19277,"style":973},[550],[296,19279,19281,19284],{"className":19280},[346],[296,19282,216],{"className":19283},[346,350],[296,19285,19287],{"className":19286},[355],[296,19288,19290,19342],{"className":19289},[359,360],[296,19291,19293,19339],{"className":19292},[364],[296,19294,19296],{"className":19295,"style":13387},[368],[296,19297,19298,19301],{"style":1005},[296,19299],{"className":19300,"style":377},[376],[296,19302,19304],{"className":19303},[381,382,383,384],[296,19305,19307],{"className":19306},[346,384],[296,19308,19310,19313],{"className":19309},[346,384],[296,19311,1711],{"className":19312},[346,350,384],[296,19314,19316],{"className":19315},[355],[296,19317,19319],{"className":19318},[359],[296,19320,19322],{"className":19321},[364],[296,19323,19325],{"className":19324,"style":13305},[368],[296,19326,19327,19330],{"style":13308},[296,19328],{"className":19329,"style":629},[376],[296,19331,19333],{"className":19332},[381,633,634,384],[296,19334,19336],{"className":19335},[346,384],[296,19337,12718],{"className":19338},[346,384],[296,19340,396],{"className":19341},[395],[296,19343,19345],{"className":19344},[364],[296,19346,19348],{"className":19347,"style":403},[368],[296,19349],{},[296,19351,1428],{"className":19352},[1539],[296,19354],{"className":19355,"style":973},[550],[296,19357,19359,19362],{"className":19358},[346],[296,19360,1422],{"className":19361},[346,350],[296,19363,19365],{"className":19364},[355],[296,19366,19368,19426],{"className":19367},[359,360],[296,19369,19371,19423],{"className":19370},[364],[296,19372,19374],{"className":19373,"style":13387},[368],[296,19375,19376,19379],{"style":1005},[296,19377],{"className":19378,"style":377},[376],[296,19380,19382],{"className":19381},[381,382,383,384],[296,19383,19385,19417,19420],{"className":19384},[346,384],[296,19386,19388,19391],{"className":19387},[346,384],[296,19389,1711],{"className":19390},[346,350,384],[296,19392,19394],{"className":19393},[355],[296,19395,19397],{"className":19396},[359],[296,19398,19400],{"className":19399},[364],[296,19401,19403],{"className":19402,"style":13305},[368],[296,19404,19405,19408],{"style":13308},[296,19406],{"className":19407,"style":629},[376],[296,19409,19411],{"className":19410},[381,633,634,384],[296,19412,19414],{"className":19413},[346,384],[296,19415,12718],{"className":19416},[346,384],[296,19418,776],{"className":19419},[877,384],[296,19421,780],{"className":19422},[346,384],[296,19424,396],{"className":19425},[395],[296,19427,19429],{"className":19428},[364],[296,19430,19432],{"className":19431,"style":890},[368],[296,19433],{},[296,19435,435],{"className":19436},[546],[296,19438],{"className":19439,"style":952},[550],[296,19441,9426],{"className":19442},[877],[296,19444],{"className":19445,"style":952},[550],[296,19447,19449,19452,19455,19458,19498],{"className":19448},[337],[296,19450],{"className":19451,"style":491},[341],[296,19453,14297],{"className":19454},[346,350],[296,19456,426],{"className":19457},[499],[296,19459,19461,19464],{"className":19460},[346],[296,19462,1422],{"className":19463},[346,350],[296,19465,19467],{"className":19466},[355],[296,19468,19470,19490],{"className":19469},[359,360],[296,19471,19473,19487],{"className":19472},[364],[296,19474,19476],{"className":19475,"style":2065},[368],[296,19477,19478,19481],{"style":1005},[296,19479],{"className":19480,"style":377},[376],[296,19482,19484],{"className":19483},[381,382,383,384],[296,19485,1711],{"className":19486},[346,350,384],[296,19488,396],{"className":19489},[395],[296,19491,19493],{"className":19492},[364],[296,19494,19496],{"className":19495,"style":403},[368],[296,19497],{},[296,19499,435],{"className":19500},[546],[212,19502,19503,19504,19573],{},"尽管方差不同，这些选择给出的策略梯度期望值全都相同。事实上，",[296,19505,19507,19524],{"className":19506},[299],[296,19508,19510],{"className":19509},[303],[305,19511,19512],{"xmlns":307},[309,19513,19514,19522],{},[312,19515,19516],{},[315,19517,19518,19520],{},[318,19519,17712],{"mathvariant":802},[318,19521,1711],{},[325,19523,18271],{"encoding":327},[296,19525,19527],{"className":19526,"ariaHidden":333},[332],[296,19528,19530,19533],{"className":19529},[337],[296,19531],{"className":19532,"style":9924},[341],[296,19534,19536,19539],{"className":19535},[346],[296,19537,17712],{"className":19538},[346],[296,19540,19542],{"className":19541},[355],[296,19543,19545,19565],{"className":19544},[359,360],[296,19546,19548,19562],{"className":19547},[364],[296,19549,19551],{"className":19550,"style":2065},[368],[296,19552,19553,19556],{"style":1005},[296,19554],{"className":19555,"style":377},[376],[296,19557,19559],{"className":19558},[381,382,383,384],[296,19560,1711],{"className":19561},[346,350,384],[296,19563,396],{"className":19564},[395],[296,19566,19568],{"className":19567},[364],[296,19569,19571],{"className":19570,"style":403},[368],[296,19572],{}," 还有两个同样合法的选择，也很重要，值得知道。",[212,19575,19576,19579],{},[247,19577,19578],{},"1. 同策略动作价值函数","。取",[296,19581,19583],{"className":19582},[751],[296,19584,19586,19635],{"className":19585},[299],[296,19587,19589],{"className":19588},[303],[305,19590,19591],{"xmlns":307,"display":760},[309,19592,19593,19632],{},[312,19594,19595,19601,19603,19614,19616,19622,19624,19630],{},[315,19596,19597,19599],{},[318,19598,17712],{"mathvariant":802},[318,19600,1711],{},[423,19602,438],{},[12712,19604,19605,19608],{},[318,19606,19607],{},"Q",[315,19609,19610,19612],{},[318,19611,320],{},[318,19613,323],{},[423,19615,426],{"stretchy":425},[315,19617,19618,19620],{},[318,19619,1422],{},[318,19621,1711],{},[423,19623,1428],{"separator":333},[315,19625,19626,19628],{},[318,19627,216],{},[318,19629,1711],{},[423,19631,435],{"stretchy":425},[325,19633,19634],{"encoding":327},"\\Phi_t = Q^{\\pi_{\\theta}}(s_t, a_t)",[296,19636,19638,19693],{"className":19637,"ariaHidden":333},[332],[296,19639,19641,19644,19684,19687,19690],{"className":19640},[337],[296,19642],{"className":19643,"style":9924},[341],[296,19645,19647,19650],{"className":19646},[346],[296,19648,17712],{"className":19649},[346],[296,19651,19653],{"className":19652},[355],[296,19654,19656,19676],{"className":19655},[359,360],[296,19657,19659,19673],{"className":19658},[364],[296,19660,19662],{"className":19661,"style":2065},[368],[296,19663,19664,19667],{"style":1005},[296,19665],{"className":19666,"style":377},[376],[296,19668,19670],{"className":19669},[381,382,383,384],[296,19671,1711],{"className":19672},[346,350,384],[296,19674,396],{"className":19675},[395],[296,19677,19679],{"className":19678},[364],[296,19680,19682],{"className":19681,"style":403},[368],[296,19683],{},[296,19685],{"className":19686,"style":551},[550],[296,19688,438],{"className":19689},[555],[296,19691],{"className":19692,"style":551},[550],[296,19694,19696,19699,19773,19776,19816,19819,19822,19862],{"className":19695},[337],[296,19697],{"className":19698,"style":491},[341],[296,19700,19702,19705],{"className":19701},[346],[296,19703,19607],{"className":19704},[346,350],[296,19706,19708],{"className":19707},[355],[296,19709,19711],{"className":19710},[359],[296,19712,19714],{"className":19713},[364],[296,19715,19718],{"className":19716,"style":19717},[368],"height:0.7144em;",[296,19719,19721,19724],{"style":19720},"top:-3.113em;margin-right:0.05em;",[296,19722],{"className":19723,"style":377},[376],[296,19725,19727],{"className":19726},[381,382,383,384],[296,19728,19730],{"className":19729},[346,384],[296,19731,19733,19736],{"className":19732},[346,384],[296,19734,320],{"className":19735,"style":351},[346,350,384],[296,19737,19739],{"className":19738},[355],[296,19740,19742,19765],{"className":19741},[359,360],[296,19743,19745,19762],{"className":19744},[364],[296,19746,19748],{"className":19747,"style":622},[368],[296,19749,19750,19753],{"style":625},[296,19751],{"className":19752,"style":629},[376],[296,19754,19756],{"className":19755},[381,633,634,384],[296,19757,19759],{"className":19758},[346,384],[296,19760,323],{"className":19761,"style":391},[346,350,384],[296,19763,396],{"className":19764},[395],[296,19766,19768],{"className":19767},[364],[296,19769,19771],{"className":19770,"style":650},[368],[296,19772],{},[296,19774,426],{"className":19775},[499],[296,19777,19779,19782],{"className":19778},[346],[296,19780,1422],{"className":19781},[346,350],[296,19783,19785],{"className":19784},[355],[296,19786,19788,19808],{"className":19787},[359,360],[296,19789,19791,19805],{"className":19790},[364],[296,19792,19794],{"className":19793,"style":2065},[368],[296,19795,19796,19799],{"style":1005},[296,19797],{"className":19798,"style":377},[376],[296,19800,19802],{"className":19801},[381,382,383,384],[296,19803,1711],{"className":19804},[346,350,384],[296,19806,396],{"className":19807},[395],[296,19809,19811],{"className":19810},[364],[296,19812,19814],{"className":19813,"style":403},[368],[296,19815],{},[296,19817,1428],{"className":19818},[1539],[296,19820],{"className":19821,"style":973},[550],[296,19823,19825,19828],{"className":19824},[346],[296,19826,216],{"className":19827},[346,350],[296,19829,19831],{"className":19830},[355],[296,19832,19834,19854],{"className":19833},[359,360],[296,19835,19837,19851],{"className":19836},[364],[296,19838,19840],{"className":19839,"style":2065},[368],[296,19841,19842,19845],{"style":1005},[296,19843],{"className":19844,"style":377},[376],[296,19846,19848],{"className":19847},[381,382,383,384],[296,19849,1711],{"className":19850},[346,350,384],[296,19852,396],{"className":19853},[395],[296,19855,19857],{"className":19856},[364],[296,19858,19860],{"className":19859,"style":403},[368],[296,19861],{},[296,19863,435],{"className":19864},[546],[212,19866,19867,19868,8685],{},"也是合法的。（选读）证明见",[216,19869,19872],{"href":19870,"rel":19871},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Fextra_pg_proof2.html",[220],"这个页面",[212,19874,19875,19878,19879,19883,19884,20333],{},[247,19876,19877],{},"2. 优势函数","。回忆一下，动作的",[216,19880,19882],{"href":19881},"\u002Fposts\u002Fspinning-up-rl-intro#%E4%BC%98%E5%8A%BF%E5%87%BD%E6%95%B0","优势","定义为 ",[296,19885,19887,19968],{"className":19886},[299],[296,19888,19890],{"className":19889},[303],[305,19891,19892],{"xmlns":307},[309,19893,19894,19965],{},[312,19895,19896,19903,19905,19911,19913,19919,19921,19923,19929,19931,19937,19939,19945,19947,19949,19955,19957,19963],{},[12712,19897,19898,19901],{},[318,19899,19900],{},"A",[318,19902,320],{},[423,19904,426],{"stretchy":425},[315,19906,19907,19909],{},[318,19908,1422],{},[318,19910,1711],{},[423,19912,1428],{"separator":333},[315,19914,19915,19917],{},[318,19916,216],{},[318,19918,1711],{},[423,19920,435],{"stretchy":425},[423,19922,438],{},[12712,19924,19925,19927],{},[318,19926,19607],{},[318,19928,320],{},[423,19930,426],{"stretchy":425},[315,19932,19933,19935],{},[318,19934,1422],{},[318,19936,1711],{},[423,19938,1428],{"separator":333},[315,19940,19941,19943],{},[318,19942,216],{},[318,19944,1711],{},[423,19946,435],{"stretchy":425},[423,19948,9426],{},[12712,19950,19951,19953],{},[318,19952,15983],{},[318,19954,320],{},[423,19956,426],{"stretchy":425},[315,19958,19959,19961],{},[318,19960,1422],{},[318,19962,1711],{},[423,19964,435],{"stretchy":425},[325,19966,19967],{"encoding":327},"A^{\\pi}(s_t,a_t) = Q^{\\pi}(s_t,a_t) - V^{\\pi}(s_t)",[296,19969,19971,20110,20249],{"className":19970,"ariaHidden":333},[332],[296,19972,19974,19977,20009,20012,20052,20055,20058,20098,20101,20104,20107],{"className":19973},[337],[296,19975],{"className":19976,"style":491},[341],[296,19978,19980,19983],{"className":19979},[346],[296,19981,19900],{"className":19982},[346,350],[296,19984,19986],{"className":19985},[355],[296,19987,19989],{"className":19988},[359],[296,19990,19992],{"className":19991},[364],[296,19993,19995],{"className":19994,"style":16026},[368],[296,19996,19997,20000],{"style":16029},[296,19998],{"className":19999,"style":377},[376],[296,20001,20003],{"className":20002},[381,382,383,384],[296,20004,20006],{"className":20005},[346,384],[296,20007,320],{"className":20008,"style":351},[346,350,384],[296,20010,426],{"className":20011},[499],[296,20013,20015,20018],{"className":20014},[346],[296,20016,1422],{"className":20017},[346,350],[296,20019,20021],{"className":20020},[355],[296,20022,20024,20044],{"className":20023},[359,360],[296,20025,20027,20041],{"className":20026},[364],[296,20028,20030],{"className":20029,"style":2065},[368],[296,20031,20032,20035],{"style":1005},[296,20033],{"className":20034,"style":377},[376],[296,20036,20038],{"className":20037},[381,382,383,384],[296,20039,1711],{"className":20040},[346,350,384],[296,20042,396],{"className":20043},[395],[296,20045,20047],{"className":20046},[364],[296,20048,20050],{"className":20049,"style":403},[368],[296,20051],{},[296,20053,1428],{"className":20054},[1539],[296,20056],{"className":20057,"style":973},[550],[296,20059,20061,20064],{"className":20060},[346],[296,20062,216],{"className":20063},[346,350],[296,20065,20067],{"className":20066},[355],[296,20068,20070,20090],{"className":20069},[359,360],[296,20071,20073,20087],{"className":20072},[364],[296,20074,20076],{"className":20075,"style":2065},[368],[296,20077,20078,20081],{"style":1005},[296,20079],{"className":20080,"style":377},[376],[296,20082,20084],{"className":20083},[381,382,383,384],[296,20085,1711],{"className":20086},[346,350,384],[296,20088,396],{"className":20089},[395],[296,20091,20093],{"className":20092},[364],[296,20094,20096],{"className":20095,"style":403},[368],[296,20097],{},[296,20099,435],{"className":20100},[546],[296,20102],{"className":20103,"style":551},[550],[296,20105,438],{"className":20106},[555],[296,20108],{"className":20109,"style":551},[550],[296,20111,20113,20116,20148,20151,20191,20194,20197,20237,20240,20243,20246],{"className":20112},[337],[296,20114],{"className":20115,"style":491},[341],[296,20117,20119,20122],{"className":20118},[346],[296,20120,19607],{"className":20121},[346,350],[296,20123,20125],{"className":20124},[355],[296,20126,20128],{"className":20127},[359],[296,20129,20131],{"className":20130},[364],[296,20132,20134],{"className":20133,"style":16026},[368],[296,20135,20136,20139],{"style":16029},[296,20137],{"className":20138,"style":377},[376],[296,20140,20142],{"className":20141},[381,382,383,384],[296,20143,20145],{"className":20144},[346,384],[296,20146,320],{"className":20147,"style":351},[346,350,384],[296,20149,426],{"className":20150},[499],[296,20152,20154,20157],{"className":20153},[346],[296,20155,1422],{"className":20156},[346,350],[296,20158,20160],{"className":20159},[355],[296,20161,20163,20183],{"className":20162},[359,360],[296,20164,20166,20180],{"className":20165},[364],[296,20167,20169],{"className":20168,"style":2065},[368],[296,20170,20171,20174],{"style":1005},[296,20172],{"className":20173,"style":377},[376],[296,20175,20177],{"className":20176},[381,382,383,384],[296,20178,1711],{"className":20179},[346,350,384],[296,20181,396],{"className":20182},[395],[296,20184,20186],{"className":20185},[364],[296,20187,20189],{"className":20188,"style":403},[368],[296,20190],{},[296,20192,1428],{"className":20193},[1539],[296,20195],{"className":20196,"style":973},[550],[296,20198,20200,20203],{"className":20199},[346],[296,20201,216],{"className":20202},[346,350],[296,20204,20206],{"className":20205},[355],[296,20207,20209,20229],{"className":20208},[359,360],[296,20210,20212,20226],{"className":20211},[364],[296,20213,20215],{"className":20214,"style":2065},[368],[296,20216,20217,20220],{"style":1005},[296,20218],{"className":20219,"style":377},[376],[296,20221,20223],{"className":20222},[381,382,383,384],[296,20224,1711],{"className":20225},[346,350,384],[296,20227,396],{"className":20228},[395],[296,20230,20232],{"className":20231},[364],[296,20233,20235],{"className":20234,"style":403},[368],[296,20236],{},[296,20238,435],{"className":20239},[546],[296,20241],{"className":20242,"style":952},[550],[296,20244,9426],{"className":20245},[877],[296,20247],{"className":20248,"style":952},[550],[296,20250,20252,20255,20287,20290,20330],{"className":20251},[337],[296,20253],{"className":20254,"style":491},[341],[296,20256,20258,20261],{"className":20257},[346],[296,20259,15983],{"className":20260,"style":952},[346,350],[296,20262,20264],{"className":20263},[355],[296,20265,20267],{"className":20266},[359],[296,20268,20270],{"className":20269},[364],[296,20271,20273],{"className":20272,"style":16026},[368],[296,20274,20275,20278],{"style":16029},[296,20276],{"className":20277,"style":377},[376],[296,20279,20281],{"className":20280},[381,382,383,384],[296,20282,20284],{"className":20283},[346,384],[296,20285,320],{"className":20286,"style":351},[346,350,384],[296,20288,426],{"className":20289},[499],[296,20291,20293,20296],{"className":20292},[346],[296,20294,1422],{"className":20295},[346,350],[296,20297,20299],{"className":20298},[355],[296,20300,20302,20322],{"className":20301},[359,360],[296,20303,20305,20319],{"className":20304},[364],[296,20306,20308],{"className":20307,"style":2065},[368],[296,20309,20310,20313],{"style":1005},[296,20311],{"className":20312,"style":377},[376],[296,20314,20316],{"className":20315},[381,382,383,384],[296,20317,1711],{"className":20318},[346,350,384],[296,20320,396],{"className":20321},[395],[296,20323,20325],{"className":20324},[364],[296,20326,20328],{"className":20327,"style":403},[368],[296,20329],{},[296,20331,435],{"className":20332},[546],"，描述的是（相对当前策略）它平均而言比其他动作好多少。取",[296,20335,20337],{"className":20336},[751],[296,20338,20340,20388],{"className":20339},[299],[296,20341,20343],{"className":20342},[303],[305,20344,20345],{"xmlns":307,"display":760},[309,20346,20347,20385],{},[312,20348,20349,20355,20357,20367,20369,20375,20377,20383],{},[315,20350,20351,20353],{},[318,20352,17712],{"mathvariant":802},[318,20354,1711],{},[423,20356,438],{},[12712,20358,20359,20361],{},[318,20360,19900],{},[315,20362,20363,20365],{},[318,20364,320],{},[318,20366,323],{},[423,20368,426],{"stretchy":425},[315,20370,20371,20373],{},[318,20372,1422],{},[318,20374,1711],{},[423,20376,1428],{"separator":333},[315,20378,20379,20381],{},[318,20380,216],{},[318,20382,1711],{},[423,20384,435],{"stretchy":425},[325,20386,20387],{"encoding":327},"\\Phi_t = A^{\\pi_{\\theta}}(s_t, a_t)",[296,20389,20391,20446],{"className":20390,"ariaHidden":333},[332],[296,20392,20394,20397,20437,20440,20443],{"className":20393},[337],[296,20395],{"className":20396,"style":9924},[341],[296,20398,20400,20403],{"className":20399},[346],[296,20401,17712],{"className":20402},[346],[296,20404,20406],{"className":20405},[355],[296,20407,20409,20429],{"className":20408},[359,360],[296,20410,20412,20426],{"className":20411},[364],[296,20413,20415],{"className":20414,"style":2065},[368],[296,20416,20417,20420],{"style":1005},[296,20418],{"className":20419,"style":377},[376],[296,20421,20423],{"className":20422},[381,382,383,384],[296,20424,1711],{"className":20425},[346,350,384],[296,20427,396],{"className":20428},[395],[296,20430,20432],{"className":20431},[364],[296,20433,20435],{"className":20434,"style":403},[368],[296,20436],{},[296,20438],{"className":20439,"style":551},[550],[296,20441,438],{"className":20442},[555],[296,20444],{"className":20445,"style":551},[550],[296,20447,20449,20452,20524,20527,20567,20570,20573,20613],{"className":20448},[337],[296,20450],{"className":20451,"style":491},[341],[296,20453,20455,20458],{"className":20454},[346],[296,20456,19900],{"className":20457},[346,350],[296,20459,20461],{"className":20460},[355],[296,20462,20464],{"className":20463},[359],[296,20465,20467],{"className":20466},[364],[296,20468,20470],{"className":20469,"style":19717},[368],[296,20471,20472,20475],{"style":19720},[296,20473],{"className":20474,"style":377},[376],[296,20476,20478],{"className":20477},[381,382,383,384],[296,20479,20481],{"className":20480},[346,384],[296,20482,20484,20487],{"className":20483},[346,384],[296,20485,320],{"className":20486,"style":351},[346,350,384],[296,20488,20490],{"className":20489},[355],[296,20491,20493,20516],{"className":20492},[359,360],[296,20494,20496,20513],{"className":20495},[364],[296,20497,20499],{"className":20498,"style":622},[368],[296,20500,20501,20504],{"style":625},[296,20502],{"className":20503,"style":629},[376],[296,20505,20507],{"className":20506},[381,633,634,384],[296,20508,20510],{"className":20509},[346,384],[296,20511,323],{"className":20512,"style":391},[346,350,384],[296,20514,396],{"className":20515},[395],[296,20517,20519],{"className":20518},[364],[296,20520,20522],{"className":20521,"style":650},[368],[296,20523],{},[296,20525,426],{"className":20526},[499],[296,20528,20530,20533],{"className":20529},[346],[296,20531,1422],{"className":20532},[346,350],[296,20534,20536],{"className":20535},[355],[296,20537,20539,20559],{"className":20538},[359,360],[296,20540,20542,20556],{"className":20541},[364],[296,20543,20545],{"className":20544,"style":2065},[368],[296,20546,20547,20550],{"style":1005},[296,20548],{"className":20549,"style":377},[376],[296,20551,20553],{"className":20552},[381,382,383,384],[296,20554,1711],{"className":20555},[346,350,384],[296,20557,396],{"className":20558},[395],[296,20560,20562],{"className":20561},[364],[296,20563,20565],{"className":20564,"style":403},[368],[296,20566],{},[296,20568,1428],{"className":20569},[1539],[296,20571],{"className":20572,"style":973},[550],[296,20574,20576,20579],{"className":20575},[346],[296,20577,216],{"className":20578},[346,350],[296,20580,20582],{"className":20581},[355],[296,20583,20585,20605],{"className":20584},[359,360],[296,20586,20588,20602],{"className":20587},[364],[296,20589,20591],{"className":20590,"style":2065},[368],[296,20592,20593,20596],{"style":1005},[296,20594],{"className":20595,"style":377},[376],[296,20597,20599],{"className":20598},[381,382,383,384],[296,20600,1711],{"className":20601},[346,350,384],[296,20603,396],{"className":20604},[395],[296,20606,20608],{"className":20607},[364],[296,20609,20611],{"className":20610,"style":403},[368],[296,20612],{},[296,20614,435],{"className":20615},[546],[212,20617,20618,20619,20897],{},"也合法。证明：它等价于先取 ",[296,20620,20622,20669],{"className":20621},[299],[296,20623,20625],{"className":20624},[303],[305,20626,20627],{"xmlns":307},[309,20628,20629,20667],{},[312,20630,20631,20637,20639,20649,20651,20657,20659,20665],{},[315,20632,20633,20635],{},[318,20634,17712],{"mathvariant":802},[318,20636,1711],{},[423,20638,438],{},[12712,20640,20641,20643],{},[318,20642,19607],{},[315,20644,20645,20647],{},[318,20646,320],{},[318,20648,323],{},[423,20650,426],{"stretchy":425},[315,20652,20653,20655],{},[318,20654,1422],{},[318,20656,1711],{},[423,20658,1428],{"separator":333},[315,20660,20661,20663],{},[318,20662,216],{},[318,20664,1711],{},[423,20666,435],{"stretchy":425},[325,20668,19634],{"encoding":327},[296,20670,20672,20727],{"className":20671,"ariaHidden":333},[332],[296,20673,20675,20678,20718,20721,20724],{"className":20674},[337],[296,20676],{"className":20677,"style":9924},[341],[296,20679,20681,20684],{"className":20680},[346],[296,20682,17712],{"className":20683},[346],[296,20685,20687],{"className":20686},[355],[296,20688,20690,20710],{"className":20689},[359,360],[296,20691,20693,20707],{"className":20692},[364],[296,20694,20696],{"className":20695,"style":2065},[368],[296,20697,20698,20701],{"style":1005},[296,20699],{"className":20700,"style":377},[376],[296,20702,20704],{"className":20703},[381,382,383,384],[296,20705,1711],{"className":20706},[346,350,384],[296,20708,396],{"className":20709},[395],[296,20711,20713],{"className":20712},[364],[296,20714,20716],{"className":20715,"style":403},[368],[296,20717],{},[296,20719],{"className":20720,"style":551},[550],[296,20722,438],{"className":20723},[555],[296,20725],{"className":20726,"style":551},[550],[296,20728,20730,20733,20805,20808,20848,20851,20854,20894],{"className":20729},[337],[296,20731],{"className":20732,"style":491},[341],[296,20734,20736,20739],{"className":20735},[346],[296,20737,19607],{"className":20738},[346,350],[296,20740,20742],{"className":20741},[355],[296,20743,20745],{"className":20744},[359],[296,20746,20748],{"className":20747},[364],[296,20749,20751],{"className":20750,"style":16026},[368],[296,20752,20753,20756],{"style":16029},[296,20754],{"className":20755,"style":377},[376],[296,20757,20759],{"className":20758},[381,382,383,384],[296,20760,20762],{"className":20761},[346,384],[296,20763,20765,20768],{"className":20764},[346,384],[296,20766,320],{"className":20767,"style":351},[346,350,384],[296,20769,20771],{"className":20770},[355],[296,20772,20774,20797],{"className":20773},[359,360],[296,20775,20777,20794],{"className":20776},[364],[296,20778,20780],{"className":20779,"style":622},[368],[296,20781,20782,20785],{"style":625},[296,20783],{"className":20784,"style":629},[376],[296,20786,20788],{"className":20787},[381,633,634,384],[296,20789,20791],{"className":20790},[346,384],[296,20792,323],{"className":20793,"style":391},[346,350,384],[296,20795,396],{"className":20796},[395],[296,20798,20800],{"className":20799},[364],[296,20801,20803],{"className":20802,"style":650},[368],[296,20804],{},[296,20806,426],{"className":20807},[499],[296,20809,20811,20814],{"className":20810},[346],[296,20812,1422],{"className":20813},[346,350],[296,20815,20817],{"className":20816},[355],[296,20818,20820,20840],{"className":20819},[359,360],[296,20821,20823,20837],{"className":20822},[364],[296,20824,20826],{"className":20825,"style":2065},[368],[296,20827,20828,20831],{"style":1005},[296,20829],{"className":20830,"style":377},[376],[296,20832,20834],{"className":20833},[381,382,383,384],[296,20835,1711],{"className":20836},[346,350,384],[296,20838,396],{"className":20839},[395],[296,20841,20843],{"className":20842},[364],[296,20844,20846],{"className":20845,"style":403},[368],[296,20847],{},[296,20849,1428],{"className":20850},[1539],[296,20852],{"className":20853,"style":973},[550],[296,20855,20857,20860],{"className":20856},[346],[296,20858,216],{"className":20859},[346,350],[296,20861,20863],{"className":20862},[355],[296,20864,20866,20886],{"className":20865},[359,360],[296,20867,20869,20883],{"className":20868},[364],[296,20870,20872],{"className":20871,"style":2065},[368],[296,20873,20874,20877],{"style":1005},[296,20875],{"className":20876,"style":377},[376],[296,20878,20880],{"className":20879},[381,382,383,384],[296,20881,1711],{"className":20882},[346,350,384],[296,20884,396],{"className":20885},[395],[296,20887,20889],{"className":20888},[364],[296,20890,20892],{"className":20891,"style":403},[368],[296,20893],{},[296,20895,435],{"className":20896},[546],"，再加一个价值函数基线，而基线我们随时可以加。",[208,20899,20900],{"title":8676},[212,20901,20902],{},"用优势函数表述策略梯度极其常见，不同算法有各自估计优势函数的办法。",[208,20904,20905,20984],{"title":8676},[212,20906,20907,20908,20913,20914,20983],{},"想更深入这个主题，应该去读 ",[216,20909,20912],{"href":20910,"rel":20911},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1506.02438",[220],"Generalized Advantage Estimation","（GAE）的论文，它的背景章节深入讨论了 ",[296,20915,20917,20934],{"className":20916},[299],[296,20918,20920],{"className":20919},[303],[305,20921,20922],{"xmlns":307},[309,20923,20924,20932],{},[312,20925,20926],{},[315,20927,20928,20930],{},[318,20929,17712],{"mathvariant":802},[318,20931,1711],{},[325,20933,18271],{"encoding":327},[296,20935,20937],{"className":20936,"ariaHidden":333},[332],[296,20938,20940,20943],{"className":20939},[337],[296,20941],{"className":20942,"style":9924},[341],[296,20944,20946,20949],{"className":20945},[346],[296,20947,17712],{"className":20948},[346],[296,20950,20952],{"className":20951},[355],[296,20953,20955,20975],{"className":20954},[359,360],[296,20956,20958,20972],{"className":20957},[364],[296,20959,20961],{"className":20960,"style":2065},[368],[296,20962,20963,20966],{"style":1005},[296,20964],{"className":20965,"style":377},[376],[296,20967,20969],{"className":20968},[381,382,383,384],[296,20970,1711],{"className":20971},[346,350,384],[296,20973,396],{"className":20974},[395],[296,20976,20978],{"className":20977},[364],[296,20979,20981],{"className":20980,"style":403},[368],[296,20982],{}," 的各种选择。",[212,20985,20986],{},"那篇论文接着提出了 GAE，一种在策略优化算法中近似优势函数的方法，如今使用非常广泛。比如 Spinning Up 的 VPG、TRPO、PPO 实现都用它。所以我们强烈建议研究它。",[289,20988,20989],{"id":20989},"小结",[212,20991,20992,20993,20996],{},"本章讲了策略梯度方法的基本理论，并把其中一些早期结果对接到了代码示例上。有兴趣的读者可以从这里继续，去研究后面的结果（价值函数基线、策略梯度的优势形式）如何落实到 Spinning Up 的 ",[216,20994,286],{"href":284,"rel":20995},[220]," 实现里。",[20998,20999,21000],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":40,"searchDepth":8707,"depth":8713,"links":21002},[21003,21004,21005,21006,21007,21008,21009,21010],{"id":291,"depth":8707,"text":291},{"id":8659,"depth":8707,"text":8659},{"id":9881,"depth":8707,"text":9882},{"id":11904,"depth":8707,"text":11904},{"id":14179,"depth":8707,"text":14180},{"id":14279,"depth":8707,"text":14279},{"id":17597,"depth":8707,"text":17597},{"id":20989,"depth":8707,"text":20989},"md",{},{"title":21014,"description":7},[6],"posts\u002Fspinning-up-policy-optimization",[11,12,13,14],"9q2H9tZjqx5pE-tgq8oO0zsMrBBqfAz3vwMgB7GvDZY",1790607246370]