[{"data":1,"prerenderedAt":15830},["ShallowReactive",2],{"blog-posts":3,"content-\u002Fposts\u002Fspinning-up-rl-intro":202},[4,16,21,26,37,43,47,59,66,72,76,81,89,96,104,109,115,121,126,131,138,145,152,158,165,173,179,185,191,197],{"path":5,"title":6,"description":7,"date":8,"categories":9,"tags":10,"draft":15},"\u002Fposts\u002Fspinning-up-policy-optimization","译","OpenAI Spinning Up 教程第三部分的全文翻译：推导最简单的策略梯度、log-derivative trick、EGLP 引理、reward-to-go 与基线，配 PyTorch 示例代码。","2026-09-28 17:49:25","技术",[11,12,13,14],"强化学习","RL","Spinning Up","翻译",false,{"path":17,"title":6,"description":18,"date":19,"categories":9,"tags":20,"draft":15},"\u002Fposts\u002Fspinning-up-rl-intro-2","OpenAI Spinning Up 第二部分的全文翻译：无模型与基于模型的 RL、策略优化与 Q-learning 的取舍，以及规划、专家迭代和经验增强等方法。","2026-09-28 10:20:21",[11,12,13,14],{"path":22,"title":6,"description":23,"date":24,"categories":9,"tags":25,"draft":15},"\u002Fposts\u002Fspinning-up-rl-intro","OpenAI Spinning Up 教程第一部分的全文翻译：智能体与环境、状态与观测、动作空间、策略、轨迹、奖励与回报、价值函数、贝尔曼方程、优势函数，附 MDP 形式化。","2026-09-28 09:29:40",[11,12,13,14],{"path":27,"title":28,"description":29,"date":30,"categories":31,"tags":32,"draft":36},"\u002Fposts\u002Fwhen-machines-come-alive","冰箱之后，生活还会怎样改变？","从厨房里的果蝇想到冰箱、移动互联网与 AI 的多轮调用：AGI 已经到来，只是昂贵的智能会怎样改变生活？","2026-09-25 06:36:22","生活杂谈",[33,34,35],"AI","随笔","科技",true,{"path":38,"title":39,"description":40,"date":41,"categories":40,"tags":42,"draft":36},"\u002Fposts\u002Fmathematical-foundations-of-llms-1","LLM的数学基础（1）：","","2026-09-19 22:59:55",null,{"path":44,"title":45,"description":40,"date":46,"categories":40,"tags":42,"draft":36},"\u002Fposts\u002Ftowards-maximize-serving-capatity","如何在 offline bench 里最大化 LLM serving system 的吞吐","2026-09-19 05:45:05",{"path":48,"title":49,"description":50,"date":51,"categories":52,"tags":53,"draft":15},"\u002Fposts\u002Fllm-memory-compute-tradeoff","LLM 的 Memory-Compute Trade-off：从 MHA 到 MLA、Sparse 与 Linear Attention","ds4.1f 上线了。它的架构 novel 在哪，便宜和快又是从哪省出来的。","2026-09-11 12:00:00","技术杂谈",[54,55,56,57,58],"LLMs","Attention","MLA","DeepSeek","长上下文",{"path":60,"title":61,"description":62,"date":63,"categories":52,"tags":64,"draft":36},"\u002Fposts\u002Fmarkdown-extensions","Markdown 扩展语法速查","本站 Markdown 支持的扩展语法演示与速查：admonition 提示块、GitHub 仓库卡片、spoiler 遮罩、ABC 乐谱、音频播放器、浏览器可运行代码块。","2026-09-01 15:30:00",[65],"博客",{"path":67,"title":68,"description":69,"date":70,"categories":31,"tags":71,"draft":15},"\u002Fposts\u002Fwhy-we-still-need-to-write-a-blog","为什么还需要手写博客？","AI这么强，还要古法写博客吗？","2026-09-01 13:17:53",[65],{"path":73,"title":74,"description":40,"date":75,"categories":40,"tags":42,"draft":36},"\u002Fposts\u002Fllm-serving-1","LLM Serving（一）:以在H200上部署GLM-5.2为例","2026-07-21 06:15:44",{"path":77,"title":78,"description":40,"date":79,"categories":80,"tags":42,"draft":15},"\u002Fposts\u002Finteresting-music-rhythm","有趣的音乐节奏","2026-07-16 12:47:24","音乐",{"path":82,"title":83,"description":84,"date":85,"categories":9,"tags":86,"draft":15},"\u002Fposts\u002Ftowards-monosemanticity","LLM 可解释性（一）：SAE与单语义","读 Anthropic《Towards Monosemanticity》。从难以解释的神经元出发, 讲清superposition为什么让单个神经元混成一团, SAE怎么把多语义的激活分解成单语义特征。","2026-07-12 22:00:00",[54,87,88],"可解释性","SAE",{"path":90,"title":91,"description":92,"date":93,"categories":31,"tags":94,"draft":36},"\u002Fposts\u002Fpiketty-ai-china-k-shaped","用皮凯蒂的镜头看 AI 与中国 K 型社会","一次凌晨的对话整理。从 r>g 出发，看 AI 时代的资本分配、中国的世袭化趋势、体制内的二元劳动力市场，以及一个普通 00 后在一线打拼的真实牌面。","2026-06-24 03:00:00",[34,95,33],"经济",{"path":97,"title":98,"description":40,"date":99,"categories":40,"tags":100,"draft":36},"\u002Fposts\u002Fclaude-code-compact-behavior","cluade code compact 行为","2026-06-23 20:17:42",[101,102,103],"Agent","AI Agent","Claude Code",{"path":105,"title":106,"description":40,"date":107,"categories":9,"tags":108,"draft":15},"\u002Fposts\u002Fclaude-code-settings","Claude Code 配置分享","2026-06-23 14:22:12",[54,102,101,103],{"path":110,"title":111,"description":112,"date":113,"categories":52,"tags":114,"draft":15},"\u002Fposts\u002Fmy-thoughts-on-the-ai-industry","简单聊聊对 AI Agent 行业的理解","写这篇文章的初衷，是我发觉逐渐忘了前两个月的 AI\u002FLLMs 是什么样的能力和情况了。意识中似乎某些在突然加速，但理性和直觉告诉我这不对，作为行内人，我知道这是一个渐进的、累积的过程。我想自己回忆一下，Agent的发展。","2026-06-20 13:14:25",[54,102,101],{"path":116,"title":117,"description":40,"date":118,"categories":9,"tags":119,"draft":15},"\u002Fposts\u002Frotational-positional-encoding","RoPE","2026-06-17 18:31:40",[54,120,117],"Transformer",{"path":122,"title":123,"description":40,"date":124,"categories":9,"tags":125,"draft":15},"\u002Fposts\u002Fintroduction-to-large-language-model","大语言模型入门基础","2026-06-16 22:36:11",[120,54],{"path":127,"title":128,"description":40,"date":129,"categories":52,"tags":130,"draft":15},"\u002Fposts\u002Fthought-about-coding-agent","关于 Coding Agent 使用的遐思","2026-06-03 17:54:20",[101],{"path":132,"title":133,"description":134,"date":135,"categories":9,"tags":136,"draft":15},"\u002Fposts\u002Fabout-ssh-usage","关于SSH的使用——客户端篇","介绍SSH客户端的用法。","2025-06-17 02:44:57",[137],"工具",{"path":139,"title":140,"description":141,"date":142,"categories":52,"tags":143,"draft":15},"\u002Fposts\u002Fopensourcesoftwarestarttutorial","如何开始一个开源项目？","浅谈我对开源项目的认知。","2025-04-17 18:00:00",[144],"开源",{"path":146,"title":147,"description":148,"date":149,"categories":150,"tags":151,"draft":15},"\u002Fposts\u002Fmotivation","渴求动力","有意识的生活并主动地行动会使得生活更美好。","2025-03-05 00:44:57","生活-情感",[34],{"path":153,"title":154,"description":155,"date":156,"categories":150,"tags":157,"draft":15},"\u002Fposts\u002F2024summary","2024年终总结","啊，痛彻的时光；啊，悲怆的人！","2025-01-01 01:00:00",[34],{"path":159,"title":160,"description":161,"date":162,"categories":9,"tags":163,"draft":15},"\u002Fposts\u002Fvps-crash-startup","快速搭建个人VPS","如何快速搭建个人VPS。","2024-12-31 19:00:00",[164],"VPS",{"path":166,"title":167,"description":168,"date":169,"categories":9,"tags":170,"draft":15},"\u002Fposts\u002Fdata-label-software","AI数据（自动）标注软件概述","对市面上目前的AI数据（自动）标注软件进行调研。","2024-12-20 14:00:00",[171,172],"数据标注","深度学习",{"path":174,"title":175,"description":176,"date":177,"categories":150,"tags":178,"draft":15},"\u002Fposts\u002Fself-destruction","生活的崩坏与自我毁灭","是什么在牵引着我？我的行为，我的思想？我的感受？我所处的地方？它们在何处？","2024-12-19 12:00:00",[34],{"path":180,"title":181,"description":182,"date":183,"categories":9,"tags":184,"draft":15},"\u002Fposts\u002Fcs194-196_llmagents","LLM Agents 课程笔记","UCB CS294\u002F194-196 Large Language Model Agents 的课程笔记。","2024-11-30 20:00:00",[101],{"path":186,"title":187,"description":188,"date":189,"categories":150,"tags":190,"draft":15},"\u002Fposts\u002Fwhatsyourworth","价值、位置","你们的生存究竟有何价值？如果毫无价值，你们究竟为何存在？","2024-11-29 02:00:00",[34],{"path":192,"title":193,"description":194,"date":195,"categories":150,"tags":196,"draft":15},"\u002Fposts\u002Fblogrelifever2","重启博客v2","忙里偷闲学习前端，还是决定结合想法手搓一个自己的博客。","2024-11-25 00:00:00",[65],{"path":198,"title":199,"description":200,"date":201,"categories":150,"tags":42,"draft":15},"\u002Fposts\u002Fblogrelife","重启博客","在我的表达欲与现实所需(问题解决记录)驱动下，我决定重写博客。","2024-10-21 20:44:57",{"id":203,"title":6,"body":204,"categories":9,"date":24,"description":23,"draft":15,"extension":15823,"meta":15824,"navigation":36,"path":22,"seo":15825,"stem":15827,"tags":15828,"__hash__":15829},"posts\u002Fposts\u002Fspinning-up-rl-intro.md",{"type":205,"value":206,"toc":15804},"minimark",[207,229,232,250,253,258,273,299,302,311,319,330,333,356,360,446,449,460,533,536,551,554,557,595,778,812,977,980,987,1055,1552,1556,1567,1617,1663,1678,1683,1686,1692,1706,1709,1832,1835,2271,4305,4308,4343,4630,4779,4930,5247,5469,5472,5702,5705,5714,5717,5751,6027,6381,6429,6436,6632,6721,6954,6967,6976,6980,6987,6990,7022,7591,7640,7938,7941,8117,8184,8186,8197,8200,10192,10201,10679,10683,10918,11114,11340,11430,11433,11440,11446,11449,12539,13169,13172,14176,14241,14246,14249,14260,14526,14772,14777,14781,14925,15791,15800],[208,209,211],"note",{"title":210},"译者说明",[212,213,214,215,222,223,228],"p",{},"本文是 OpenAI ",[216,217,221],"a",{"href":218,"rel":219},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002F",[220],"nofollow","Spinning Up in Deep RL"," 教程第一部分 ",[216,224,227],{"href":225,"rel":226},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Frl_intro.html",[220],"Part 1: Key Concepts in RL"," 的全文翻译，原仓库以 MIT 协议发布。原文的提示框、外链与代码示例全部保留，视频以链接代替嵌入。术语首次出现时附英文原词，之后沿用同一译名。",[212,230,231],{},"欢迎来到强化学习导论！这一部分的目标是让你熟悉：",[233,234,235,239,247],"ul",{},[236,237,238],"li",{},"讨论这一主题所用的语言与记号；",[236,240,241,242,246],{},"RL 算法在做什么的高层解释（不过我们大多回避它们",[243,244,245],"em",{},"如何","做到这一问题）；",[236,248,249],{},"以及支撑这些算法的一点点核心数学。",[212,251,252],{},"概括地说，RL 研究的是智能体（agent）以及它们如何通过试错（trial and error）学习。它把这个想法形式化：对智能体的行为给予奖励或惩罚，会让它未来更倾向于重复或放弃该行为。",[254,255,257],"h2",{"id":256},"rl-能做什么","RL 能做什么",[212,259,260,261,266,267,272],{},"近年来，RL 方法收获了大范围的成功。例如，它被用来教计算机在模拟中控制机器人：",[216,262,265],{"href":263,"rel":264},"https:\u002F\u002Fd4mucfpksywv.cloudfront.net\u002Fopenai-baselines-ppo\u002Fknocked-over-stand-up.mp4",[220],"模拟环境里，机器人被撞倒后重新站起来（视频）","；也在真实世界中这样做：",[216,268,271],{"href":269,"rel":270},"https:\u002F\u002Fwww.youtube.com\u002Fwatch?v=jwSbzNHGflM",[220],"真实机器人控制（YouTube）","。",[212,274,275,276,281,282,287,288,293,294,272],{},"它还因在复杂策略游戏中造出突破性 AI 而闻名，最有名的是 ",[216,277,280],{"href":278,"rel":279},"https:\u002F\u002Fdeepmind.com\u002Fresearch\u002Falphago\u002F",[220],"Go（围棋）","和 ",[216,283,286],{"href":284,"rel":285},"https:\u002F\u002Fblog.openai.com\u002Fopenai-five\u002F",[220],"Dota","；它教会计算机从原始像素",[216,289,292],{"href":290,"rel":291},"https:\u002F\u002Fdeepmind.com\u002Fresearch\u002Fdqn\u002F",[220],"玩 Atari 游戏","，也训练过模拟机器人",[216,295,298],{"href":296,"rel":297},"https:\u002F\u002Fblog.openai.com\u002Fdeep-reinforcement-learning-from-human-preferences\u002F",[220],"跟随人类指令",[254,300,301],{"id":301},"关键概念与术语",[212,303,304],{},[305,306],"img",{"alt":307,"className":308,"src":310},"智能体与环境的交互循环",[309],"dark-ink","\u002Fimages\u002Fspinning-up-rl-loop.png",[212,312,313,314,318],{},"RL 的两个主角是",[315,316,317],"strong",{},"智能体（agent）和环境（environment）","。环境是智能体身处其中并与之交互的世界。交互的每一步，智能体都会看到世界状态的一个（可能不完整的）观测，然后决定要采取的动作。智能体作用于环境时，环境会随之改变，但它也可能自行变化。",[212,320,321,322,325,326,329],{},"智能体还会从环境感知到一个",[315,323,324],{},"奖励（reward）"," 信号：一个数字，告诉它当前的世界状态有多好或多坏。智能体的目标是最大化自己的累积奖励，这个累积量称为",[315,327,328],{},"回报（return）","。强化学习方法，就是智能体学习行为以达成这一目标的各种途径。",[212,331,332],{},"要把 RL 做的事说得更具体，还需要引入一批术语。我们要讨论：",[233,334,335,338,341,344,347,350,353],{},[236,336,337],{},"状态与观测；",[236,339,340],{},"动作空间；",[236,342,343],{},"策略；",[236,345,346],{},"轨迹；",[236,348,349],{},"回报的不同形式；",[236,351,352],{},"RL 优化问题；",[236,354,355],{},"以及价值函数。",[357,358,359],"h3",{"id":359},"状态与观测",[212,361,362,365,366,412,413,365,416,445],{},[315,363,364],{},"状态（state）"," ",[367,368,371,393],"span",{"className":369},[370],"katex",[367,372,375],{"className":373},[374],"katex-mathml",[376,377,379],"math",{"xmlns":378},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[380,381,382,389],"semantics",{},[383,384,385],"mrow",{},[386,387,388],"mi",{},"s",[390,391,388],"annotation",{"encoding":392},"application\u002Fx-tex",[367,394,398],{"className":395,"ariaHidden":397},[396],"katex-html","true",[367,399,402,407],{"className":400},[401],"base",[367,403],{"className":404,"style":406},[405],"strut","height:0.4306em;",[367,408,388],{"className":409},[410,411],"mord","mathnormal"," 是对世界的完整描述，没有任何关于世界的信息被它遗漏。",[315,414,415],{},"观测（observation）",[367,417,419,433],{"className":418},[370],[367,420,422],{"className":421},[374],[376,423,424],{"xmlns":378},[380,425,426,431],{},[383,427,428],{},[386,429,430],{},"o",[390,432,430],{"encoding":392},[367,434,436],{"className":435,"ariaHidden":397},[396],[367,437,439,442],{"className":438},[401],[367,440],{"className":441,"style":406},[405],[367,443,430],{"className":444},[410,411]," 则是对状态的部分描述，可能省略一些信息。",[212,447,448],{},"在深度 RL 中，我们几乎总是用实值向量、矩阵或更高阶张量来表示状态和观测。例如，一个视觉观测可以用其像素值的 RGB（红绿蓝）矩阵表示；机器人的状态可以用它的关节角度和速度表示。",[212,450,451,452,455,456,459],{},"当智能体能够观测到环境的完整状态时，我们说环境是",[315,453,454],{},"完全可观测（fully observed）"," 的；当它只能看到部分观测时，我们说环境是",[315,457,458],{},"部分可观测（partially observed）"," 的。",[208,461,463,524],{"title":462},"你应该知道",[212,464,465,466,494,495,523],{},"RL 的记号有时会在严格来说更适合写观测符号 ",[367,467,469,482],{"className":468},[370],[367,470,472],{"className":471},[374],[376,473,474],{"xmlns":378},[380,475,476,480],{},[383,477,478],{},[386,479,430],{},[390,481,430],{"encoding":392},[367,483,485],{"className":484,"ariaHidden":397},[396],[367,486,488,491],{"className":487},[401],[367,489],{"className":490,"style":406},[405],[367,492,430],{"className":493},[410,411]," 的地方写状态符号 ",[367,496,498,511],{"className":497},[370],[367,499,501],{"className":500},[374],[376,502,503],{"xmlns":378},[380,504,505,509],{},[383,506,507],{},[386,508,388],{},[390,510,388],{"encoding":392},[367,512,514],{"className":513,"ariaHidden":397},[396],[367,515,517,520],{"className":516},[401],[367,518],{"className":519,"style":406},[405],[367,521,388],{"className":522},[410,411],"。描述智能体如何决定动作时就常见这种情况：记号上通常写作“动作以状态为条件”，但实践中动作取决于观测，因为智能体拿不到状态。",[212,525,526,527,532],{},"本指南遵循标准记号约定，具体指哪个应当能从上下文分辨出来。如果有不清楚的地方，欢迎到",[216,528,531],{"href":529,"rel":530},"https:\u002F\u002Fgithub.com\u002Fopenai\u002Fspinningup\u002Fissues",[220],"原仓库","提 issue！我们的目标是把人教会，而不是把人绕晕。",[357,534,535],{"id":535},"动作空间",[212,537,538,539,542,543,546,547,550],{},"不同的环境允许不同种类的动作。一个给定环境中所有合法动作的集合，通常叫作",[315,540,541],{},"动作空间（action space）","。有些环境（比如 Atari 和围棋）具有",[315,544,545],{},"离散动作空间（discrete action space）","：智能体可用的走法只有有限多个。另一些环境（比如智能体在物理世界中控制机器人）则具有",[315,548,549],{},"连续动作空间（continuous action space）","：动作是实值向量。",[212,552,553],{},"这个区分对深度 RL 方法的影响相当深远：有些算法家族只能直接用于其中一种情形，要用到另一种情形上就得动大手术。",[357,555,556],{"id":556},"策略",[212,558,559,562,563,594],{},[315,560,561],{},"策略（policy）"," 是智能体用来决定采取什么动作的规则。它可以是确定型（deterministic）的，通常记作 ",[367,564,566,581],{"className":565},[370],[367,567,569],{"className":568},[374],[376,570,571],{"xmlns":378},[380,572,573,578],{},[383,574,575],{},[386,576,577],{},"μ",[390,579,580],{"encoding":392},"\\mu",[367,582,584],{"className":583,"ariaHidden":397},[396],[367,585,587,591],{"className":586},[401],[367,588],{"className":589,"style":590},[405],"height:0.625em;vertical-align:-0.1944em;",[367,592,577],{"className":593},[410,411],"：",[367,596,599],{"className":597},[598],"katex-display",[367,600,602,642],{"className":601},[370],[367,603,605],{"className":604},[374],[376,606,608],{"xmlns":378,"display":607},"block",[380,609,610,639],{},[383,611,612,620,624,626,630,636],{},[613,614,615,617],"msub",{},[386,616,216],{},[386,618,619],{},"t",[621,622,623],"mo",{},"=",[386,625,577],{},[621,627,629],{"stretchy":628},"false","(",[613,631,632,634],{},[386,633,388],{},[386,635,619],{},[621,637,638],{"stretchy":628},")",[390,640,641],{"encoding":392},"a_t = \\mu(s_t)",[367,643,645,720],{"className":644,"ariaHidden":397},[396],[367,646,648,652,708,713,717],{"className":647},[401],[367,649],{"className":650,"style":651},[405],"height:0.5806em;vertical-align:-0.15em;",[367,653,655,658],{"className":654},[410],[367,656,216],{"className":657},[410,411],[367,659,662],{"className":660},[661],"msupsub",[367,663,667,699],{"className":664},[665,666],"vlist-t","vlist-t2",[367,668,671,694],{"className":669},[670],"vlist-r",[367,672,676],{"className":673,"style":675},[674],"vlist","height:0.2806em;",[367,677,679,684],{"style":678},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[367,680],{"className":681,"style":683},[682],"pstrut","height:2.7em;",[367,685,691],{"className":686},[687,688,689,690],"sizing","reset-size6","size3","mtight",[367,692,619],{"className":693},[410,411,690],[367,695,698],{"className":696},[697],"vlist-s","​",[367,700,702],{"className":701},[670],[367,703,706],{"className":704,"style":705},[674],"height:0.15em;",[367,707],{},[367,709],{"className":710,"style":712},[711],"mspace","margin-right:0.2778em;",[367,714,623],{"className":715},[716],"mrel",[367,718],{"className":719,"style":712},[711],[367,721,723,727,730,734,774],{"className":722},[401],[367,724],{"className":725,"style":726},[405],"height:1em;vertical-align:-0.25em;",[367,728,577],{"className":729},[410,411],[367,731,629],{"className":732},[733],"mopen",[367,735,737,740],{"className":736},[410],[367,738,388],{"className":739},[410,411],[367,741,743],{"className":742},[661],[367,744,746,766],{"className":745},[665,666],[367,747,749,763],{"className":748},[670],[367,750,752],{"className":751,"style":675},[674],[367,753,754,757],{"style":678},[367,755],{"className":756,"style":683},[682],[367,758,760],{"className":759},[687,688,689,690],[367,761,619],{"className":762},[410,411,690],[367,764,698],{"className":765},[697],[367,767,769],{"className":768},[670],[367,770,772],{"className":771,"style":705},[674],[367,773],{},[367,775,638],{"className":776},[777],"mclose",[212,779,780,781,594],{},"也可以是随机型（stochastic）的，通常记作 ",[367,782,784,799],{"className":783},[370],[367,785,787],{"className":786},[374],[376,788,789],{"xmlns":378},[380,790,791,796],{},[383,792,793],{},[386,794,795],{},"π",[390,797,798],{"encoding":392},"\\pi",[367,800,802],{"className":801,"ariaHidden":397},[396],[367,803,805,808],{"className":804},[401],[367,806],{"className":807,"style":406},[405],[367,809,795],{"className":810,"style":811},[410,411],"margin-right:0.0359em;",[367,813,815],{"className":814},[598],[367,816,818,858],{"className":817},[370],[367,819,821],{"className":820},[374],[376,822,823],{"xmlns":378,"display":607},[380,824,825,855],{},[383,826,827,833,836,838,840,843,847,853],{},[613,828,829,831],{},[386,830,216],{},[386,832,619],{},[621,834,835],{},"∼",[386,837,795],{},[621,839,629],{"stretchy":628},[621,841,842],{},"⋅",[386,844,846],{"mathvariant":845},"normal","∣",[613,848,849,851],{},[386,850,388],{},[386,852,619],{},[621,854,638],{"stretchy":628},[390,856,857],{"encoding":392},"a_t \\sim \\pi(\\cdot | s_t)",[367,859,861,916],{"className":860,"ariaHidden":397},[396],[367,862,864,867,907,910,913],{"className":863},[401],[367,865],{"className":866,"style":651},[405],[367,868,870,873],{"className":869},[410],[367,871,216],{"className":872},[410,411],[367,874,876],{"className":875},[661],[367,877,879,899],{"className":878},[665,666],[367,880,882,896],{"className":881},[670],[367,883,885],{"className":884,"style":675},[674],[367,886,887,890],{"style":678},[367,888],{"className":889,"style":683},[682],[367,891,893],{"className":892},[687,688,689,690],[367,894,619],{"className":895},[410,411,690],[367,897,698],{"className":898},[697],[367,900,902],{"className":901},[670],[367,903,905],{"className":904,"style":705},[674],[367,906],{},[367,908],{"className":909,"style":712},[711],[367,911,835],{"className":912},[716],[367,914],{"className":915,"style":712},[711],[367,917,919,922,925,928,931,934,974],{"className":918},[401],[367,920],{"className":921,"style":726},[405],[367,923,795],{"className":924,"style":811},[410,411],[367,926,629],{"className":927},[733],[367,929,842],{"className":930},[410],[367,932,846],{"className":933},[410],[367,935,937,940],{"className":936},[410],[367,938,388],{"className":939},[410,411],[367,941,943],{"className":942},[661],[367,944,946,966],{"className":945},[665,666],[367,947,949,963],{"className":948},[670],[367,950,952],{"className":951,"style":675},[674],[367,953,954,957],{"style":678},[367,955],{"className":956,"style":683},[682],[367,958,960],{"className":959},[687,688,689,690],[367,961,619],{"className":962},[410,411,690],[367,964,698],{"className":965},[697],[367,967,969],{"className":968},[670],[367,970,972],{"className":971,"style":705},[674],[367,973],{},[367,975,638],{"className":976},[777],[212,978,979],{},"策略是智能体的大脑，所以拿「policy」一词替换「agent」来说话并不少见，比如『策略正在试图最大化奖励』。",[212,981,982,983,986],{},"在深度 RL 中，我们处理的是",[315,984,985],{},"参数化策略（parameterized policy）","：这类策略的输出是可计算函数，函数依赖一组参数（比如神经网络的权重和偏置）；通过某种优化算法调整这些参数，就能改变策略的行为。",[212,988,989,990,1022,1023,1054],{},"这类策略的参数常记作 ",[367,991,993,1008],{"className":992},[370],[367,994,996],{"className":995},[374],[376,997,998],{"xmlns":378},[380,999,1000,1005],{},[383,1001,1002],{},[386,1003,1004],{},"θ",[390,1006,1007],{"encoding":392},"\\theta",[367,1009,1011],{"className":1010,"ariaHidden":397},[396],[367,1012,1014,1018],{"className":1013},[401],[367,1015],{"className":1016,"style":1017},[405],"height:0.6944em;",[367,1019,1004],{"className":1020,"style":1021},[410,411],"margin-right:0.0278em;"," 或 ",[367,1024,1026,1041],{"className":1025},[370],[367,1027,1029],{"className":1028},[374],[376,1030,1031],{"xmlns":378},[380,1032,1033,1038],{},[383,1034,1035],{},[386,1036,1037],{},"ϕ",[390,1039,1040],{"encoding":392},"\\phi",[367,1042,1044],{"className":1043,"ariaHidden":397},[396],[367,1045,1047,1051],{"className":1046},[401],[367,1048],{"className":1049,"style":1050},[405],"height:0.8889em;vertical-align:-0.1944em;",[367,1052,1037],{"className":1053},[410,411],"，并写成策略符号的下标来点明这层关系：",[367,1056,1058],{"className":1057},[598],[367,1059,1061,1161],{"className":1060},[370],[367,1062,1064],{"className":1063},[374],[376,1065,1066],{"xmlns":378,"display":607},[380,1067,1068,1158],{},[1069,1070,1074,1116],"mtable",{"rowspacing":1071,"columnalign":1072,"columnspacing":1073},"0.25em","right left","0em",[1075,1076,1077,1090],"mtr",{},[1078,1079,1080],"mtd",{},[1081,1082,1084],"mstyle",{"scriptlevel":1083,"displaystyle":397},"0",[613,1085,1086,1088],{},[386,1087,216],{},[386,1089,619],{},[1078,1091,1092],{},[1081,1093,1094],{"scriptlevel":1083,"displaystyle":397},[383,1095,1096,1098,1100,1106,1108,1114],{},[383,1097],{},[621,1099,623],{},[613,1101,1102,1104],{},[386,1103,577],{},[386,1105,1004],{},[621,1107,629],{"stretchy":628},[613,1109,1110,1112],{},[386,1111,388],{},[386,1113,619],{},[621,1115,638],{"stretchy":628},[1075,1117,1118,1128],{},[1078,1119,1120],{},[1081,1121,1122],{"scriptlevel":1083,"displaystyle":397},[613,1123,1124,1126],{},[386,1125,216],{},[386,1127,619],{},[1078,1129,1130],{},[1081,1131,1132],{"scriptlevel":1083,"displaystyle":397},[383,1133,1134,1136,1138,1144,1146,1148,1150,1156],{},[383,1135],{},[621,1137,835],{},[613,1139,1140,1142],{},[386,1141,795],{},[386,1143,1004],{},[621,1145,629],{"stretchy":628},[621,1147,842],{},[386,1149,846],{"mathvariant":845},[613,1151,1152,1154],{},[386,1153,388],{},[386,1155,619],{},[621,1157,638],{"stretchy":628},[390,1159,1160],{"encoding":392},"\\begin{aligned}\na_t &= \\mu_{\\theta}(s_t) \\\\\na_t &\\sim \\pi_{\\theta}(\\cdot | s_t)\n\\end{aligned}",[367,1162,1164],{"className":1163,"ariaHidden":397},[396],[367,1165,1167,1171],{"className":1166},[401],[367,1168],{"className":1169,"style":1170},[405],"height:2.7em;vertical-align:-1.1em;",[367,1172,1174],{"className":1173},[410],[367,1175,1177,1302],{"className":1176},[1069],[367,1178,1181],{"className":1179},[1180],"col-align-r",[367,1182,1184,1293],{"className":1183},[665,666],[367,1185,1187,1290],{"className":1186},[670],[367,1188,1191,1241],{"className":1189,"style":1190},[674],"height:1.6em;",[367,1192,1194,1198],{"style":1193},"top:-3.76em;",[367,1195],{"className":1196,"style":1197},[682],"height:3em;",[367,1199,1201],{"className":1200},[410],[367,1202,1204,1207],{"className":1203},[410],[367,1205,216],{"className":1206},[410,411],[367,1208,1210],{"className":1209},[661],[367,1211,1213,1233],{"className":1212},[665,666],[367,1214,1216,1230],{"className":1215},[670],[367,1217,1219],{"className":1218,"style":675},[674],[367,1220,1221,1224],{"style":678},[367,1222],{"className":1223,"style":683},[682],[367,1225,1227],{"className":1226},[687,688,689,690],[367,1228,619],{"className":1229},[410,411,690],[367,1231,698],{"className":1232},[697],[367,1234,1236],{"className":1235},[670],[367,1237,1239],{"className":1238,"style":705},[674],[367,1240],{},[367,1242,1244,1247],{"style":1243},"top:-2.26em;",[367,1245],{"className":1246,"style":1197},[682],[367,1248,1250],{"className":1249},[410],[367,1251,1253,1256],{"className":1252},[410],[367,1254,216],{"className":1255},[410,411],[367,1257,1259],{"className":1258},[661],[367,1260,1262,1282],{"className":1261},[665,666],[367,1263,1265,1279],{"className":1264},[670],[367,1266,1268],{"className":1267,"style":675},[674],[367,1269,1270,1273],{"style":678},[367,1271],{"className":1272,"style":683},[682],[367,1274,1276],{"className":1275},[687,688,689,690],[367,1277,619],{"className":1278},[410,411,690],[367,1280,698],{"className":1281},[697],[367,1283,1285],{"className":1284},[670],[367,1286,1288],{"className":1287,"style":705},[674],[367,1289],{},[367,1291,698],{"className":1292},[697],[367,1294,1296],{"className":1295},[670],[367,1297,1300],{"className":1298,"style":1299},[674],"height:1.1em;",[367,1301],{},[367,1303,1306],{"className":1304},[1305],"col-align-l",[367,1307,1309,1544],{"className":1308},[665,666],[367,1310,1312,1541],{"className":1311},[670],[367,1313,1315,1425],{"className":1314,"style":1190},[674],[367,1316,1317,1320],{"style":1193},[367,1318],{"className":1319,"style":1197},[682],[367,1321,1323,1326,1329,1332,1335,1379,1382,1422],{"className":1322},[410],[367,1324],{"className":1325},[410],[367,1327],{"className":1328,"style":712},[711],[367,1330,623],{"className":1331},[716],[367,1333],{"className":1334,"style":712},[711],[367,1336,1338,1341],{"className":1337},[410],[367,1339,577],{"className":1340},[410,411],[367,1342,1344],{"className":1343},[661],[367,1345,1347,1371],{"className":1346},[665,666],[367,1348,1350,1368],{"className":1349},[670],[367,1351,1354],{"className":1352,"style":1353},[674],"height:0.3361em;",[367,1355,1356,1359],{"style":678},[367,1357],{"className":1358,"style":683},[682],[367,1360,1362],{"className":1361},[687,688,689,690],[367,1363,1365],{"className":1364},[410,690],[367,1366,1004],{"className":1367,"style":1021},[410,411,690],[367,1369,698],{"className":1370},[697],[367,1372,1374],{"className":1373},[670],[367,1375,1377],{"className":1376,"style":705},[674],[367,1378],{},[367,1380,629],{"className":1381},[733],[367,1383,1385,1388],{"className":1384},[410],[367,1386,388],{"className":1387},[410,411],[367,1389,1391],{"className":1390},[661],[367,1392,1394,1414],{"className":1393},[665,666],[367,1395,1397,1411],{"className":1396},[670],[367,1398,1400],{"className":1399,"style":675},[674],[367,1401,1402,1405],{"style":678},[367,1403],{"className":1404,"style":683},[682],[367,1406,1408],{"className":1407},[687,688,689,690],[367,1409,619],{"className":1410},[410,411,690],[367,1412,698],{"className":1413},[697],[367,1415,1417],{"className":1416},[670],[367,1418,1420],{"className":1419,"style":705},[674],[367,1421],{},[367,1423,638],{"className":1424},[777],[367,1426,1427,1430],{"style":1243},[367,1428],{"className":1429,"style":1197},[682],[367,1431,1433,1436,1439,1442,1445,1489,1492,1495,1498,1538],{"className":1432},[410],[367,1434],{"className":1435},[410],[367,1437],{"className":1438,"style":712},[711],[367,1440,835],{"className":1441},[716],[367,1443],{"className":1444,"style":712},[711],[367,1446,1448,1451],{"className":1447},[410],[367,1449,795],{"className":1450,"style":811},[410,411],[367,1452,1454],{"className":1453},[661],[367,1455,1457,1481],{"className":1456},[665,666],[367,1458,1460,1478],{"className":1459},[670],[367,1461,1463],{"className":1462,"style":1353},[674],[367,1464,1466,1469],{"style":1465},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[367,1467],{"className":1468,"style":683},[682],[367,1470,1472],{"className":1471},[687,688,689,690],[367,1473,1475],{"className":1474},[410,690],[367,1476,1004],{"className":1477,"style":1021},[410,411,690],[367,1479,698],{"className":1480},[697],[367,1482,1484],{"className":1483},[670],[367,1485,1487],{"className":1486,"style":705},[674],[367,1488],{},[367,1490,629],{"className":1491},[733],[367,1493,842],{"className":1494},[410],[367,1496,846],{"className":1497},[410],[367,1499,1501,1504],{"className":1500},[410],[367,1502,388],{"className":1503},[410,411],[367,1505,1507],{"className":1506},[661],[367,1508,1510,1530],{"className":1509},[665,666],[367,1511,1513,1527],{"className":1512},[670],[367,1514,1516],{"className":1515,"style":675},[674],[367,1517,1518,1521],{"style":678},[367,1519],{"className":1520,"style":683},[682],[367,1522,1524],{"className":1523},[687,688,689,690],[367,1525,619],{"className":1526},[410,411,690],[367,1528,698],{"className":1529},[697],[367,1531,1533],{"className":1532},[670],[367,1534,1536],{"className":1535,"style":705},[674],[367,1537],{},[367,1539,638],{"className":1540},[777],[367,1542,698],{"className":1543},[697],[367,1545,1547],{"className":1546},[670],[367,1548,1550],{"className":1549,"style":1299},[674],[367,1551],{},[1553,1554,1555],"h4",{"id":1555},"确定型策略",[212,1557,1558,1561,1562,1566],{},[315,1559,1560],{},"例子：确定型策略","。下面这段代码用 PyTorch 的 ",[1563,1564,1565],"code",{},"torch.nn"," 包，为连续动作空间构建一个简单的确定型策略：",[1568,1569,1573],"pre",{"className":1570,"code":1571,"language":1572,"meta":40,"style":40},"language-python3 shiki shiki-themes github-light github-dark","pi_net = nn.Sequential(\n              nn.Linear(obs_dim, 64),\n              nn.Tanh(),\n              nn.Linear(64, 64),\n              nn.Tanh(),\n              nn.Linear(64, act_dim)\n            )\n","python3",[1563,1574,1575,1582,1588,1594,1600,1605,1611],{"__ignoreMap":40},[367,1576,1579],{"class":1577,"line":1578},"line",1,[367,1580,1581],{},"pi_net = nn.Sequential(\n",[367,1583,1585],{"class":1577,"line":1584},2,[367,1586,1587],{},"              nn.Linear(obs_dim, 64),\n",[367,1589,1591],{"class":1577,"line":1590},3,[367,1592,1593],{},"              nn.Tanh(),\n",[367,1595,1597],{"class":1577,"line":1596},4,[367,1598,1599],{},"              nn.Linear(64, 64),\n",[367,1601,1603],{"class":1577,"line":1602},5,[367,1604,1593],{},[367,1606,1608],{"class":1577,"line":1607},6,[367,1609,1610],{},"              nn.Linear(64, act_dim)\n",[367,1612,1614],{"class":1577,"line":1613},7,[367,1615,1616],{},"            )\n",[212,1618,1619,1620,1654,1655,1658,1659,1662],{},"它构建了一个 MLP（multi-layer perceptron，多层感知机）网络：两个隐藏层各 64 个单元，激活函数是 ",[367,1621,1623,1641],{"className":1622},[370],[367,1624,1626],{"className":1625},[374],[376,1627,1628],{"xmlns":378},[380,1629,1630,1638],{},[383,1631,1632,1635],{},[386,1633,1634],{},"tanh",[621,1636,1637],{},"⁡",[390,1639,1640],{"encoding":392},"\\tanh",[367,1642,1644],{"className":1643,"ariaHidden":397},[396],[367,1645,1647,1650],{"className":1646},[401],[367,1648],{"className":1649,"style":1017},[405],[367,1651,1634],{"className":1652},[1653],"mop","。如果 ",[1563,1656,1657],{},"obs"," 是装着一批观测的 Numpy 数组，可以这样用 ",[1563,1660,1661],{},"pi_net"," 求出这批观测对应的动作：",[1568,1664,1666],{"className":1570,"code":1665,"language":1572,"meta":40,"style":40},"obs_tensor = torch.as_tensor(obs, dtype=torch.float32)\nactions = pi_net(obs_tensor)\n",[1563,1667,1668,1673],{"__ignoreMap":40},[367,1669,1670],{"class":1577,"line":1578},[367,1671,1672],{},"obs_tensor = torch.as_tensor(obs, dtype=torch.float32)\n",[367,1674,1675],{"class":1577,"line":1584},[367,1676,1677],{},"actions = pi_net(obs_tensor)\n",[208,1679,1680],{"title":462},[212,1681,1682],{},"神经网络的内容如果不熟悉，不必担心：本教程聚焦 RL，不展开神经网络那一侧。可以先跳过这个例子，之后再回来看。不过我们猜想，如果你已经了解，它会有所帮助。",[1553,1684,1685],{"id":1685},"随机型策略",[212,1687,1688,1689,272],{},"深度 RL 中最常见的两种随机型策略是",[315,1690,1691],{},"分类策略（categorical policy）和对角高斯策略（diagonal Gaussian policy）",[212,1693,1694,1699,1700,1705],{},[216,1695,1698],{"href":1696,"rel":1697},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FCategorical_distribution",[220],"分类","策略可用于离散动作空间，对角",[216,1701,1704],{"href":1702,"rel":1703},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FMultivariate_normal_distribution",[220],"高斯","策略则用于连续动作空间。",[212,1707,1708],{},"使用和训练随机型策略时，有两项关键计算居于中心地位：",[233,1710,1711,1714],{},[236,1712,1713],{},"从策略中采样动作；",[236,1715,1716,1717,272],{},"计算特定动作的对数似然（log likelihood）",[367,1718,1720,1753],{"className":1719},[370],[367,1721,1723],{"className":1722},[374],[376,1724,1725],{"xmlns":378},[380,1726,1727,1750],{},[383,1728,1729,1732,1734,1740,1742,1744,1746,1748],{},[386,1730,1731],{},"log",[621,1733,1637],{},[613,1735,1736,1738],{},[386,1737,795],{},[386,1739,1004],{},[621,1741,629],{"stretchy":628},[386,1743,216],{},[386,1745,846],{"mathvariant":845},[386,1747,388],{},[621,1749,638],{"stretchy":628},[390,1751,1752],{"encoding":392},"\\log \\pi_{\\theta}(a|s)",[367,1754,1756],{"className":1755,"ariaHidden":397},[396],[367,1757,1759,1762,1770,1774,1817,1820,1823,1826,1829],{"className":1758},[401],[367,1760],{"className":1761,"style":726},[405],[367,1763,1765,1766],{"className":1764},[1653],"lo",[367,1767,1769],{"style":1768},"margin-right:0.0139em;","g",[367,1771],{"className":1772,"style":1773},[711],"margin-right:0.1667em;",[367,1775,1777,1780],{"className":1776},[410],[367,1778,795],{"className":1779,"style":811},[410,411],[367,1781,1783],{"className":1782},[661],[367,1784,1786,1809],{"className":1785},[665,666],[367,1787,1789,1806],{"className":1788},[670],[367,1790,1792],{"className":1791,"style":1353},[674],[367,1793,1794,1797],{"style":1465},[367,1795],{"className":1796,"style":683},[682],[367,1798,1800],{"className":1799},[687,688,689,690],[367,1801,1803],{"className":1802},[410,690],[367,1804,1004],{"className":1805,"style":1021},[410,411,690],[367,1807,698],{"className":1808},[697],[367,1810,1812],{"className":1811},[670],[367,1813,1815],{"className":1814,"style":705},[674],[367,1816],{},[367,1818,629],{"className":1819},[733],[367,1821,216],{"className":1822},[410,411],[367,1824,846],{"className":1825},[410],[367,1827,388],{"className":1828},[410,411],[367,1830,638],{"className":1831},[777],[212,1833,1834],{},"接下来分别说明这两件事在分类策略和对角高斯策略上怎么做。",[208,1836,1838,1847,1873,2000],{"title":1837},"分类策略",[212,1839,1840,1841,1846],{},"分类策略就像作用在离散动作上的分类器。构建分类策略的神经网络和构建分类器是同一套做法：输入观测，接若干层（视输入类型可以是卷积层或全连接层），最后一个线性层给出每个动作的 logits，再接一个 ",[216,1842,1845],{"href":1843,"rel":1844},"https:\u002F\u002Fdevelopers.google.com\u002Fmachine-learning\u002Fcrash-course\u002Fmulti-class-neural-networks\u002Fsoftmax",[220],"softmax"," 把 logits 转成概率。",[212,1848,1849,1852,1853,1858,1859,1858,1866,1022,1869,1872],{},[315,1850,1851],{},"采样","。拿到每个动作的概率后，PyTorch 和 TensorFlow 这类框架都内置了采样工具，例如 ",[216,1854,1857],{"href":1855,"rel":1856},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Fdistributions.html#categorical",[220],"PyTorch 的 Categorical 分布","、",[216,1860,1863],{"href":1861,"rel":1862},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Ftorch.html#torch.multinomial",[220],[1563,1864,1865],{},"torch.multinomial",[1563,1867,1868],{},"tf.distributions.Categorical",[1563,1870,1871],{},"tf.multinomial"," 的文档。",[212,1874,1875,1878,1879,1970,1971,1999],{},[315,1876,1877],{},"对数似然","。记最后一层输出的概率为 ",[367,1880,1882,1907],{"className":1881},[370],[367,1883,1885],{"className":1884},[374],[376,1886,1887],{"xmlns":378},[380,1888,1889,1904],{},[383,1890,1891,1898,1900,1902],{},[613,1892,1893,1896],{},[386,1894,1895],{},"P",[386,1897,1004],{},[621,1899,629],{"stretchy":628},[386,1901,388],{},[621,1903,638],{"stretchy":628},[390,1905,1906],{"encoding":392},"P_{\\theta}(s)",[367,1908,1910],{"className":1909,"ariaHidden":397},[396],[367,1911,1913,1916,1961,1964,1967],{"className":1912},[401],[367,1914],{"className":1915,"style":726},[405],[367,1917,1919,1923],{"className":1918},[410],[367,1920,1895],{"className":1921,"style":1922},[410,411],"margin-right:0.1389em;",[367,1924,1926],{"className":1925},[661],[367,1927,1929,1953],{"className":1928},[665,666],[367,1930,1932,1950],{"className":1931},[670],[367,1933,1935],{"className":1934,"style":1353},[674],[367,1936,1938,1941],{"style":1937},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[367,1939],{"className":1940,"style":683},[682],[367,1942,1944],{"className":1943},[687,688,689,690],[367,1945,1947],{"className":1946},[410,690],[367,1948,1004],{"className":1949,"style":1021},[410,411,690],[367,1951,698],{"className":1952},[697],[367,1954,1956],{"className":1955},[670],[367,1957,1959],{"className":1958,"style":705},[674],[367,1960],{},[367,1962,629],{"className":1963},[733],[367,1965,388],{"className":1966},[410,411],[367,1968,638],{"className":1969},[777],"。它是一个向量，条目数与动作数相同，因此可以把动作当作向量的索引。动作 ",[367,1972,1974,1987],{"className":1973},[370],[367,1975,1977],{"className":1976},[374],[376,1978,1979],{"xmlns":378},[380,1980,1981,1985],{},[383,1982,1983],{},[386,1984,216],{},[390,1986,216],{"encoding":392},[367,1988,1990],{"className":1989,"ariaHidden":397},[396],[367,1991,1993,1996],{"className":1992},[401],[367,1994],{"className":1995,"style":406},[405],[367,1997,216],{"className":1998},[410,411]," 的对数似然通过索引得到：",[367,2001,2003],{"className":2002},[598],[367,2004,2006,2068],{"className":2005},[370],[367,2007,2009],{"className":2008},[374],[376,2010,2011],{"xmlns":378,"display":607},[380,2012,2013,2065],{},[383,2014,2015,2017,2019,2025,2027,2029,2031,2033,2035,2037,2039,2041],{},[386,2016,1731],{},[621,2018,1637],{},[613,2020,2021,2023],{},[386,2022,795],{},[386,2024,1004],{},[621,2026,629],{"stretchy":628},[386,2028,216],{},[386,2030,846],{"mathvariant":845},[386,2032,388],{},[621,2034,638],{"stretchy":628},[621,2036,623],{},[386,2038,1731],{},[621,2040,1637],{},[613,2042,2043,2063],{},[383,2044,2045,2048,2054,2056,2058,2060],{},[621,2046,2047],{"fence":397},"[",[613,2049,2050,2052],{},[386,2051,1895],{},[386,2053,1004],{},[621,2055,629],{"stretchy":628},[386,2057,388],{},[621,2059,638],{"stretchy":628},[621,2061,2062],{"fence":397},"]",[386,2064,216],{},[390,2066,2067],{"encoding":392},"\\log \\pi_{\\theta}(a|s) = \\log \\left[P_{\\theta}(s)\\right]_a",[367,2069,2071,2152],{"className":2070,"ariaHidden":397},[396],[367,2072,2074,2077,2082,2085,2128,2131,2134,2137,2140,2143,2146,2149],{"className":2073},[401],[367,2075],{"className":2076,"style":726},[405],[367,2078,1765,2080],{"className":2079},[1653],[367,2081,1769],{"style":1768},[367,2083],{"className":2084,"style":1773},[711],[367,2086,2088,2091],{"className":2087},[410],[367,2089,795],{"className":2090,"style":811},[410,411],[367,2092,2094],{"className":2093},[661],[367,2095,2097,2120],{"className":2096},[665,666],[367,2098,2100,2117],{"className":2099},[670],[367,2101,2103],{"className":2102,"style":1353},[674],[367,2104,2105,2108],{"style":1465},[367,2106],{"className":2107,"style":683},[682],[367,2109,2111],{"className":2110},[687,688,689,690],[367,2112,2114],{"className":2113},[410,690],[367,2115,1004],{"className":2116,"style":1021},[410,411,690],[367,2118,698],{"className":2119},[697],[367,2121,2123],{"className":2122},[670],[367,2124,2126],{"className":2125,"style":705},[674],[367,2127],{},[367,2129,629],{"className":2130},[733],[367,2132,216],{"className":2133},[410,411],[367,2135,846],{"className":2136},[410],[367,2138,388],{"className":2139},[410,411],[367,2141,638],{"className":2142},[777],[367,2144],{"className":2145,"style":712},[711],[367,2147,623],{"className":2148},[716],[367,2150],{"className":2151,"style":712},[711],[367,2153,2155,2159,2164,2167],{"className":2154},[401],[367,2156],{"className":2157,"style":2158},[405],"height:1.0497em;vertical-align:-0.2997em;",[367,2160,1765,2162],{"className":2161},[1653],[367,2163,1769],{"style":1768},[367,2165],{"className":2166,"style":1773},[711],[367,2168,2171,2234],{"className":2169},[2170],"minner",[367,2172,2174,2179,2222,2225,2228,2231],{"className":2173},[2170],[367,2175,2047],{"className":2176,"style":2178},[733,2177],"delimcenter","top:0em;",[367,2180,2182,2185],{"className":2181},[410],[367,2183,1895],{"className":2184,"style":1922},[410,411],[367,2186,2188],{"className":2187},[661],[367,2189,2191,2214],{"className":2190},[665,666],[367,2192,2194,2211],{"className":2193},[670],[367,2195,2197],{"className":2196,"style":1353},[674],[367,2198,2199,2202],{"style":1937},[367,2200],{"className":2201,"style":683},[682],[367,2203,2205],{"className":2204},[687,688,689,690],[367,2206,2208],{"className":2207},[410,690],[367,2209,1004],{"className":2210,"style":1021},[410,411,690],[367,2212,698],{"className":2213},[697],[367,2215,2217],{"className":2216},[670],[367,2218,2220],{"className":2219,"style":705},[674],[367,2221],{},[367,2223,629],{"className":2224},[733],[367,2226,388],{"className":2227},[410,411],[367,2229,638],{"className":2230},[777],[367,2232,2062],{"className":2233,"style":2178},[777,2177],[367,2235,2237],{"className":2236},[661],[367,2238,2240,2262],{"className":2239},[665,666],[367,2241,2243,2259],{"className":2242},[670],[367,2244,2247],{"className":2245,"style":2246},[674],"height:0.0017em;",[367,2248,2250,2253],{"style":2249},"top:-2.4003em;margin-right:0.05em;",[367,2251],{"className":2252,"style":683},[682],[367,2254,2256],{"className":2255},[687,688,689,690],[367,2257,216],{"className":2258},[410,411,690],[367,2260,698],{"className":2261},[697],[367,2263,2265],{"className":2264},[670],[367,2266,2269],{"className":2267,"style":2268},[674],"height:0.2997em;",[367,2270],{},[208,2272,2274,2338,2430,2524,2631,2696,2996,3216,3272,3559],{"title":2273},"对角高斯策略",[212,2275,2276,2277,2305,2306,2337],{},"多元高斯分布（multivariate Gaussian distribution，也叫多元正态分布）由均值向量 ",[367,2278,2280,2293],{"className":2279},[370],[367,2281,2283],{"className":2282},[374],[376,2284,2285],{"xmlns":378},[380,2286,2287,2291],{},[383,2288,2289],{},[386,2290,577],{},[390,2292,580],{"encoding":392},[367,2294,2296],{"className":2295,"ariaHidden":397},[396],[367,2297,2299,2302],{"className":2298},[401],[367,2300],{"className":2301,"style":590},[405],[367,2303,577],{"className":2304},[410,411]," 和协方差矩阵 ",[367,2307,2309,2324],{"className":2308},[370],[367,2310,2312],{"className":2311},[374],[376,2313,2314],{"xmlns":378},[380,2315,2316,2321],{},[383,2317,2318],{},[386,2319,2320],{"mathvariant":845},"Σ",[390,2322,2323],{"encoding":392},"\\Sigma",[367,2325,2327],{"className":2326,"ariaHidden":397},[396],[367,2328,2330,2334],{"className":2329},[401],[367,2331],{"className":2332,"style":2333},[405],"height:0.6833em;",[367,2335,2320],{"className":2336},[410]," 描述。对角高斯分布是协方差矩阵只有对角线元素的特例，因此可以用一个向量来表示它。",[212,2339,2340,2341,2429],{},"对角高斯策略总有一个从观测映射到均值动作 ",[367,2342,2344,2368],{"className":2343},[370],[367,2345,2347],{"className":2346},[374],[376,2348,2349],{"xmlns":378},[380,2350,2351,2365],{},[383,2352,2353,2359,2361,2363],{},[613,2354,2355,2357],{},[386,2356,577],{},[386,2358,1004],{},[621,2360,629],{"stretchy":628},[386,2362,388],{},[621,2364,638],{"stretchy":628},[390,2366,2367],{"encoding":392},"\\mu_{\\theta}(s)",[367,2369,2371],{"className":2370,"ariaHidden":397},[396],[367,2372,2374,2377,2420,2423,2426],{"className":2373},[401],[367,2375],{"className":2376,"style":726},[405],[367,2378,2380,2383],{"className":2379},[410],[367,2381,577],{"className":2382},[410,411],[367,2384,2386],{"className":2385},[661],[367,2387,2389,2412],{"className":2388},[665,666],[367,2390,2392,2409],{"className":2391},[670],[367,2393,2395],{"className":2394,"style":1353},[674],[367,2396,2397,2400],{"style":678},[367,2398],{"className":2399,"style":683},[682],[367,2401,2403],{"className":2402},[687,688,689,690],[367,2404,2406],{"className":2405},[410,690],[367,2407,1004],{"className":2408,"style":1021},[410,411,690],[367,2410,698],{"className":2411},[697],[367,2413,2415],{"className":2414},[670],[367,2416,2418],{"className":2417,"style":705},[674],[367,2419],{},[367,2421,629],{"className":2422},[733],[367,2424,388],{"className":2425},[410,411],[367,2427,638],{"className":2428},[777]," 的神经网络。协方差矩阵通常有两种表示方式。",[212,2431,2432,2435,2436,2478,2479,2482,2483,2523],{},[315,2433,2434],{},"第一种","：只有一个对数标准差向量 ",[367,2437,2439,2458],{"className":2438},[370],[367,2440,2442],{"className":2441},[374],[376,2443,2444],{"xmlns":378},[380,2445,2446,2455],{},[383,2447,2448,2450,2452],{},[386,2449,1731],{},[621,2451,1637],{},[386,2453,2454],{},"σ",[390,2456,2457],{"encoding":392},"\\log \\sigma",[367,2459,2461],{"className":2460,"ariaHidden":397},[396],[367,2462,2464,2467,2472,2475],{"className":2463},[401],[367,2465],{"className":2466,"style":1050},[405],[367,2468,1765,2470],{"className":2469},[1653],[367,2471,1769],{"style":1768},[367,2473],{"className":2474,"style":1773},[711],[367,2476,2454],{"className":2477,"style":811},[410,411],"，它",[315,2480,2481],{},"不是","状态的函数：这些 ",[367,2484,2486,2503],{"className":2485},[370],[367,2487,2489],{"className":2488},[374],[376,2490,2491],{"xmlns":378},[380,2492,2493,2501],{},[383,2494,2495,2497,2499],{},[386,2496,1731],{},[621,2498,1637],{},[386,2500,2454],{},[390,2502,2457],{"encoding":392},[367,2504,2506],{"className":2505,"ariaHidden":397},[396],[367,2507,2509,2512,2517,2520],{"className":2508},[401],[367,2510],{"className":2511,"style":1050},[405],[367,2513,1765,2515],{"className":2514},[1653],[367,2516,1769],{"style":1768},[367,2518],{"className":2519,"style":1773},[711],[367,2521,2454],{"className":2522,"style":811},[410,411]," 是独立的参数。（你应该知道：我们的 VPG（Vanilla Policy Gradient）、TRPO（Trust Region Policy Optimization）和 PPO 的实现用的就是这一种。）",[212,2525,2526,2529,2530,2630],{},[315,2527,2528],{},"第二种","：有一个从状态映射到对数标准差 ",[367,2531,2533,2561],{"className":2532},[370],[367,2534,2536],{"className":2535},[374],[376,2537,2538],{"xmlns":378},[380,2539,2540,2558],{},[383,2541,2542,2544,2546,2552,2554,2556],{},[386,2543,1731],{},[621,2545,1637],{},[613,2547,2548,2550],{},[386,2549,2454],{},[386,2551,1004],{},[621,2553,629],{"stretchy":628},[386,2555,388],{},[621,2557,638],{"stretchy":628},[390,2559,2560],{"encoding":392},"\\log \\sigma_{\\theta}(s)",[367,2562,2564],{"className":2563,"ariaHidden":397},[396],[367,2565,2567,2570,2575,2578,2621,2624,2627],{"className":2566},[401],[367,2568],{"className":2569,"style":726},[405],[367,2571,1765,2573],{"className":2572},[1653],[367,2574,1769],{"style":1768},[367,2576],{"className":2577,"style":1773},[711],[367,2579,2581,2584],{"className":2580},[410],[367,2582,2454],{"className":2583,"style":811},[410,411],[367,2585,2587],{"className":2586},[661],[367,2588,2590,2613],{"className":2589},[665,666],[367,2591,2593,2610],{"className":2592},[670],[367,2594,2596],{"className":2595,"style":1353},[674],[367,2597,2598,2601],{"style":1465},[367,2599],{"className":2600,"style":683},[682],[367,2602,2604],{"className":2603},[687,688,689,690],[367,2605,2607],{"className":2606},[410,690],[367,2608,1004],{"className":2609,"style":1021},[410,411,690],[367,2611,698],{"className":2612},[697],[367,2614,2616],{"className":2615},[670],[367,2617,2619],{"className":2618,"style":705},[674],[367,2620],{},[367,2622,629],{"className":2623},[733],[367,2625,388],{"className":2626},[410,411],[367,2628,638],{"className":2629},[777]," 的神经网络，它可以选择性地与均值网络共享一些层。",[212,2632,2633,2634,2695],{},"注意，两种情况下输出的都是对数标准差，并非直接输出标准差。原因是对数标准差可以在 ",[367,2635,2637,2664],{"className":2636},[370],[367,2638,2640],{"className":2639},[374],[376,2641,2642],{"xmlns":378},[380,2643,2644,2661],{},[383,2645,2646,2648,2651,2654,2657,2659],{},[621,2647,629],{"stretchy":628},[621,2649,2650],{},"−",[386,2652,2653],{"mathvariant":845},"∞",[621,2655,2656],{"separator":397},",",[386,2658,2653],{"mathvariant":845},[621,2660,638],{"stretchy":628},[390,2662,2663],{"encoding":392},"(-\\infty, \\infty)",[367,2665,2667],{"className":2666,"ariaHidden":397},[396],[367,2668,2670,2673,2676,2679,2682,2686,2689,2692],{"className":2669},[401],[367,2671],{"className":2672,"style":726},[405],[367,2674,629],{"className":2675},[733],[367,2677,2650],{"className":2678},[410],[367,2680,2653],{"className":2681},[410],[367,2683,2656],{"className":2684},[2685],"mpunct",[367,2687],{"className":2688,"style":1773},[711],[367,2690,2653],{"className":2691},[410],[367,2693,638],{"className":2694},[777]," 上自由取值，而标准差必须非负；不必强制这类约束，参数训练起来更容易。对对数标准差取指数即可立即得到标准差，所以这种表示不损失任何东西。",[212,2697,2698,2700,2701,2788,2789,2877,2878,2908,2909,2995],{},[315,2699,1851],{},"。给定均值动作 ",[367,2702,2704,2727],{"className":2703},[370],[367,2705,2707],{"className":2706},[374],[376,2708,2709],{"xmlns":378},[380,2710,2711,2725],{},[383,2712,2713,2719,2721,2723],{},[613,2714,2715,2717],{},[386,2716,577],{},[386,2718,1004],{},[621,2720,629],{"stretchy":628},[386,2722,388],{},[621,2724,638],{"stretchy":628},[390,2726,2367],{"encoding":392},[367,2728,2730],{"className":2729,"ariaHidden":397},[396],[367,2731,2733,2736,2779,2782,2785],{"className":2732},[401],[367,2734],{"className":2735,"style":726},[405],[367,2737,2739,2742],{"className":2738},[410],[367,2740,577],{"className":2741},[410,411],[367,2743,2745],{"className":2744},[661],[367,2746,2748,2771],{"className":2747},[665,666],[367,2749,2751,2768],{"className":2750},[670],[367,2752,2754],{"className":2753,"style":1353},[674],[367,2755,2756,2759],{"style":678},[367,2757],{"className":2758,"style":683},[682],[367,2760,2762],{"className":2761},[687,688,689,690],[367,2763,2765],{"className":2764},[410,690],[367,2766,1004],{"className":2767,"style":1021},[410,411,690],[367,2769,698],{"className":2770},[697],[367,2772,2774],{"className":2773},[670],[367,2775,2777],{"className":2776,"style":705},[674],[367,2778],{},[367,2780,629],{"className":2781},[733],[367,2783,388],{"className":2784},[410,411],[367,2786,638],{"className":2787},[777],"、标准差 ",[367,2790,2792,2816],{"className":2791},[370],[367,2793,2795],{"className":2794},[374],[376,2796,2797],{"xmlns":378},[380,2798,2799,2813],{},[383,2800,2801,2807,2809,2811],{},[613,2802,2803,2805],{},[386,2804,2454],{},[386,2806,1004],{},[621,2808,629],{"stretchy":628},[386,2810,388],{},[621,2812,638],{"stretchy":628},[390,2814,2815],{"encoding":392},"\\sigma_{\\theta}(s)",[367,2817,2819],{"className":2818,"ariaHidden":397},[396],[367,2820,2822,2825,2868,2871,2874],{"className":2821},[401],[367,2823],{"className":2824,"style":726},[405],[367,2826,2828,2831],{"className":2827},[410],[367,2829,2454],{"className":2830,"style":811},[410,411],[367,2832,2834],{"className":2833},[661],[367,2835,2837,2860],{"className":2836},[665,666],[367,2838,2840,2857],{"className":2839},[670],[367,2841,2843],{"className":2842,"style":1353},[674],[367,2844,2845,2848],{"style":1465},[367,2846],{"className":2847,"style":683},[682],[367,2849,2851],{"className":2850},[687,688,689,690],[367,2852,2854],{"className":2853},[410,690],[367,2855,1004],{"className":2856,"style":1021},[410,411,690],[367,2858,698],{"className":2859},[697],[367,2861,2863],{"className":2862},[670],[367,2864,2866],{"className":2865,"style":705},[674],[367,2867],{},[367,2869,629],{"className":2870},[733],[367,2872,388],{"className":2873},[410,411],[367,2875,638],{"className":2876},[777],"，以及一个来自球形高斯的噪声向量 ",[367,2879,2881,2895],{"className":2880},[370],[367,2882,2884],{"className":2883},[374],[376,2885,2886],{"xmlns":378},[380,2887,2888,2893],{},[383,2889,2890],{},[386,2891,2892],{},"z",[390,2894,2892],{"encoding":392},[367,2896,2898],{"className":2897,"ariaHidden":397},[396],[367,2899,2901,2904],{"className":2900},[401],[367,2902],{"className":2903,"style":406},[405],[367,2905,2892],{"className":2906,"style":2907},[410,411],"margin-right:0.044em;","（",[367,2910,2912,2944],{"className":2911},[370],[367,2913,2915],{"className":2914},[374],[376,2916,2917],{"xmlns":378},[380,2918,2919,2941],{},[383,2920,2921,2923,2925,2929,2931,2934,2936,2939],{},[386,2922,2892],{},[621,2924,835],{},[386,2926,2928],{"mathvariant":2927},"script","N",[621,2930,629],{"stretchy":628},[2932,2933,1083],"mn",{},[621,2935,2656],{"separator":397},[386,2937,2938],{},"I",[621,2940,638],{"stretchy":628},[390,2942,2943],{"encoding":392},"z \\sim \\mathcal{N}(0, I)",[367,2945,2947,2965],{"className":2946,"ariaHidden":397},[396],[367,2948,2950,2953,2956,2959,2962],{"className":2949},[401],[367,2951],{"className":2952,"style":406},[405],[367,2954,2892],{"className":2955,"style":2907},[410,411],[367,2957],{"className":2958,"style":712},[711],[367,2960,835],{"className":2961},[716],[367,2963],{"className":2964,"style":712},[711],[367,2966,2968,2971,2976,2979,2982,2985,2988,2992],{"className":2967},[401],[367,2969],{"className":2970,"style":726},[405],[367,2972,2928],{"className":2973,"style":2975},[410,2974],"mathcal","margin-right:0.1474em;",[367,2977,629],{"className":2978},[733],[367,2980,1083],{"className":2981},[410],[367,2983,2656],{"className":2984},[2685],[367,2986],{"className":2987,"style":1773},[711],[367,2989,2938],{"className":2990,"style":2991},[410,411],"margin-right:0.0785em;",[367,2993,638],{"className":2994},[777],"），动作样本可以这样计算：",[367,2997,2999],{"className":2998},[598],[367,3000,3002,3050],{"className":3001},[370],[367,3003,3005],{"className":3004},[374],[376,3006,3007],{"xmlns":378,"display":607},[380,3008,3009,3047],{},[383,3010,3011,3013,3015,3021,3023,3025,3027,3030,3036,3038,3040,3042,3045],{},[386,3012,216],{},[621,3014,623],{},[613,3016,3017,3019],{},[386,3018,577],{},[386,3020,1004],{},[621,3022,629],{"stretchy":628},[386,3024,388],{},[621,3026,638],{"stretchy":628},[621,3028,3029],{},"+",[613,3031,3032,3034],{},[386,3033,2454],{},[386,3035,1004],{},[621,3037,629],{"stretchy":628},[386,3039,388],{},[621,3041,638],{"stretchy":628},[621,3043,3044],{},"⊙",[386,3046,2892],{},[390,3048,3049],{"encoding":392},"a = \\mu_{\\theta}(s) + \\sigma_{\\theta}(s) \\odot z",[367,3051,3053,3071,3140,3207],{"className":3052,"ariaHidden":397},[396],[367,3054,3056,3059,3062,3065,3068],{"className":3055},[401],[367,3057],{"className":3058,"style":406},[405],[367,3060,216],{"className":3061},[410,411],[367,3063],{"className":3064,"style":712},[711],[367,3066,623],{"className":3067},[716],[367,3069],{"className":3070,"style":712},[711],[367,3072,3074,3077,3120,3123,3126,3129,3133,3137],{"className":3073},[401],[367,3075],{"className":3076,"style":726},[405],[367,3078,3080,3083],{"className":3079},[410],[367,3081,577],{"className":3082},[410,411],[367,3084,3086],{"className":3085},[661],[367,3087,3089,3112],{"className":3088},[665,666],[367,3090,3092,3109],{"className":3091},[670],[367,3093,3095],{"className":3094,"style":1353},[674],[367,3096,3097,3100],{"style":678},[367,3098],{"className":3099,"style":683},[682],[367,3101,3103],{"className":3102},[687,688,689,690],[367,3104,3106],{"className":3105},[410,690],[367,3107,1004],{"className":3108,"style":1021},[410,411,690],[367,3110,698],{"className":3111},[697],[367,3113,3115],{"className":3114},[670],[367,3116,3118],{"className":3117,"style":705},[674],[367,3119],{},[367,3121,629],{"className":3122},[733],[367,3124,388],{"className":3125},[410,411],[367,3127,638],{"className":3128},[777],[367,3130],{"className":3131,"style":3132},[711],"margin-right:0.2222em;",[367,3134,3029],{"className":3135},[3136],"mbin",[367,3138],{"className":3139,"style":3132},[711],[367,3141,3143,3146,3189,3192,3195,3198,3201,3204],{"className":3142},[401],[367,3144],{"className":3145,"style":726},[405],[367,3147,3149,3152],{"className":3148},[410],[367,3150,2454],{"className":3151,"style":811},[410,411],[367,3153,3155],{"className":3154},[661],[367,3156,3158,3181],{"className":3157},[665,666],[367,3159,3161,3178],{"className":3160},[670],[367,3162,3164],{"className":3163,"style":1353},[674],[367,3165,3166,3169],{"style":1465},[367,3167],{"className":3168,"style":683},[682],[367,3170,3172],{"className":3171},[687,688,689,690],[367,3173,3175],{"className":3174},[410,690],[367,3176,1004],{"className":3177,"style":1021},[410,411,690],[367,3179,698],{"className":3180},[697],[367,3182,3184],{"className":3183},[670],[367,3185,3187],{"className":3186,"style":705},[674],[367,3188],{},[367,3190,629],{"className":3191},[733],[367,3193,388],{"className":3194},[410,411],[367,3196,638],{"className":3197},[777],[367,3199],{"className":3200,"style":3132},[711],[367,3202,3044],{"className":3203},[3136],[367,3205],{"className":3206,"style":3132},[711],[367,3208,3210,3213],{"className":3209},[401],[367,3211],{"className":3212,"style":406},[405],[367,3214,2892],{"className":3215,"style":2907},[410,411],[212,3217,3218,3219,3249,3250,1022,3257,3260,3261,1022,3268,3271],{},"其中 ",[367,3220,3222,3236],{"className":3221},[370],[367,3223,3225],{"className":3224},[374],[376,3226,3227],{"xmlns":378},[380,3228,3229,3233],{},[383,3230,3231],{},[621,3232,3044],{},[390,3234,3235],{"encoding":392},"\\odot",[367,3237,3239],{"className":3238,"ariaHidden":397},[396],[367,3240,3242,3246],{"className":3241},[401],[367,3243],{"className":3244,"style":3245},[405],"height:0.6667em;vertical-align:-0.0833em;",[367,3247,3044],{"className":3248},[410]," 表示两个向量的逐元素乘积。主流框架都有内置的方式生成噪声向量，比如 ",[216,3251,3254],{"href":3252,"rel":3253},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Ftorch.html#torch.normal",[220],[1563,3255,3256],{},"torch.normal",[1563,3258,3259],{},"tf.random_normal","；也可以构建分布对象，比如 ",[216,3262,3265],{"href":3263,"rel":3264},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Fdistributions.html#normal",[220],[1563,3266,3267],{},"torch.distributions.Normal",[1563,3269,3270],{},"tf.distributions.Normal","，再用它们生成样本。（后一种方式的好处是，这些对象还能替你算对数似然。）",[212,3273,3274,3276,3277,3387,3388,3498,3499,3529,3530,3558],{},[315,3275,1877],{},"。对均值为 ",[367,3278,3280,3308],{"className":3279},[370],[367,3281,3283],{"className":3282},[374],[376,3284,3285],{"xmlns":378},[380,3286,3287,3305],{},[383,3288,3289,3291,3293,3299,3301,3303],{},[386,3290,577],{},[621,3292,623],{},[613,3294,3295,3297],{},[386,3296,577],{},[386,3298,1004],{},[621,3300,629],{"stretchy":628},[386,3302,388],{},[621,3304,638],{"stretchy":628},[390,3306,3307],{"encoding":392},"\\mu = \\mu_{\\theta}(s)",[367,3309,3311,3329],{"className":3310,"ariaHidden":397},[396],[367,3312,3314,3317,3320,3323,3326],{"className":3313},[401],[367,3315],{"className":3316,"style":590},[405],[367,3318,577],{"className":3319},[410,411],[367,3321],{"className":3322,"style":712},[711],[367,3324,623],{"className":3325},[716],[367,3327],{"className":3328,"style":712},[711],[367,3330,3332,3335,3378,3381,3384],{"className":3331},[401],[367,3333],{"className":3334,"style":726},[405],[367,3336,3338,3341],{"className":3337},[410],[367,3339,577],{"className":3340},[410,411],[367,3342,3344],{"className":3343},[661],[367,3345,3347,3370],{"className":3346},[665,666],[367,3348,3350,3367],{"className":3349},[670],[367,3351,3353],{"className":3352,"style":1353},[674],[367,3354,3355,3358],{"style":678},[367,3356],{"className":3357,"style":683},[682],[367,3359,3361],{"className":3360},[687,688,689,690],[367,3362,3364],{"className":3363},[410,690],[367,3365,1004],{"className":3366,"style":1021},[410,411,690],[367,3368,698],{"className":3369},[697],[367,3371,3373],{"className":3372},[670],[367,3374,3376],{"className":3375,"style":705},[674],[367,3377],{},[367,3379,629],{"className":3380},[733],[367,3382,388],{"className":3383},[410,411],[367,3385,638],{"className":3386},[777],"、标准差为 ",[367,3389,3391,3419],{"className":3390},[370],[367,3392,3394],{"className":3393},[374],[376,3395,3396],{"xmlns":378},[380,3397,3398,3416],{},[383,3399,3400,3402,3404,3410,3412,3414],{},[386,3401,2454],{},[621,3403,623],{},[613,3405,3406,3408],{},[386,3407,2454],{},[386,3409,1004],{},[621,3411,629],{"stretchy":628},[386,3413,388],{},[621,3415,638],{"stretchy":628},[390,3417,3418],{"encoding":392},"\\sigma = \\sigma_{\\theta}(s)",[367,3420,3422,3440],{"className":3421,"ariaHidden":397},[396],[367,3423,3425,3428,3431,3434,3437],{"className":3424},[401],[367,3426],{"className":3427,"style":406},[405],[367,3429,2454],{"className":3430,"style":811},[410,411],[367,3432],{"className":3433,"style":712},[711],[367,3435,623],{"className":3436},[716],[367,3438],{"className":3439,"style":712},[711],[367,3441,3443,3446,3489,3492,3495],{"className":3442},[401],[367,3444],{"className":3445,"style":726},[405],[367,3447,3449,3452],{"className":3448},[410],[367,3450,2454],{"className":3451,"style":811},[410,411],[367,3453,3455],{"className":3454},[661],[367,3456,3458,3481],{"className":3457},[665,666],[367,3459,3461,3478],{"className":3460},[670],[367,3462,3464],{"className":3463,"style":1353},[674],[367,3465,3466,3469],{"style":1465},[367,3467],{"className":3468,"style":683},[682],[367,3470,3472],{"className":3471},[687,688,689,690],[367,3473,3475],{"className":3474},[410,690],[367,3476,1004],{"className":3477,"style":1021},[410,411,690],[367,3479,698],{"className":3480},[697],[367,3482,3484],{"className":3483},[670],[367,3485,3487],{"className":3486,"style":705},[674],[367,3488],{},[367,3490,629],{"className":3491},[733],[367,3493,388],{"className":3494},[410,411],[367,3496,638],{"className":3497},[777]," 的对角高斯，",[367,3500,3502,3516],{"className":3501},[370],[367,3503,3505],{"className":3504},[374],[376,3506,3507],{"xmlns":378},[380,3508,3509,3514],{},[383,3510,3511],{},[386,3512,3513],{},"k",[390,3515,3513],{"encoding":392},[367,3517,3519],{"className":3518,"ariaHidden":397},[396],[367,3520,3522,3525],{"className":3521},[401],[367,3523],{"className":3524,"style":1017},[405],[367,3526,3513],{"className":3527,"style":3528},[410,411],"margin-right:0.0315em;"," 维动作 ",[367,3531,3533,3546],{"className":3532},[370],[367,3534,3536],{"className":3535},[374],[376,3537,3538],{"xmlns":378},[380,3539,3540,3544],{},[383,3541,3542],{},[386,3543,216],{},[390,3545,216],{"encoding":392},[367,3547,3549],{"className":3548,"ariaHidden":397},[396],[367,3550,3552,3555],{"className":3551},[401],[367,3553],{"className":3554,"style":406},[405],[367,3556,216],{"className":3557},[410,411]," 的对数似然为",[367,3560,3562],{"className":3561},[598],[367,3563,3565,3701],{"className":3564},[370],[367,3566,3568],{"className":3567},[374],[376,3569,3570],{"xmlns":378,"display":607},[380,3571,3572,3698],{},[383,3573,3574,3576,3578,3584,3586,3588,3590,3592,3594,3596,3598,3607],{},[386,3575,1731],{},[621,3577,1637],{},[613,3579,3580,3582],{},[386,3581,795],{},[386,3583,1004],{},[621,3585,629],{"stretchy":628},[386,3587,216],{},[386,3589,846],{"mathvariant":845},[386,3591,388],{},[621,3593,638],{"stretchy":628},[621,3595,623],{},[621,3597,2650],{},[3599,3600,3601,3604],"mfrac",{},[2932,3602,3603],{},"1",[2932,3605,3606],{},"2",[383,3608,3609,3611,3628,3684,3686,3688,3690,3692,3694,3696],{},[621,3610,629],{"fence":397},[3612,3613,3614,3617,3626],"munderover",{},[621,3615,3616],{},"∑",[383,3618,3619,3622,3624],{},[386,3620,3621],{},"i",[621,3623,623],{},[2932,3625,3603],{},[386,3627,3513],{},[383,3629,3630,3632,3668,3670,3672,3674,3676,3682],{},[621,3631,629],{"fence":397},[3599,3633,3634,3659],{},[383,3635,3636,3638,3644,3646,3652],{},[621,3637,629],{"stretchy":628},[613,3639,3640,3642],{},[386,3641,216],{},[386,3643,3621],{},[621,3645,2650],{},[613,3647,3648,3650],{},[386,3649,577],{},[386,3651,3621],{},[3653,3654,3655,3657],"msup",{},[621,3656,638],{"stretchy":628},[2932,3658,3606],{},[3660,3661,3662,3664,3666],"msubsup",{},[386,3663,2454],{},[386,3665,3621],{},[2932,3667,3606],{},[621,3669,3029],{},[2932,3671,3606],{},[386,3673,1731],{},[621,3675,1637],{},[613,3677,3678,3680],{},[386,3679,2454],{},[386,3681,3621],{},[621,3683,638],{"fence":397},[621,3685,3029],{},[386,3687,3513],{},[386,3689,1731],{},[621,3691,1637],{},[2932,3693,3606],{},[386,3695,795],{},[621,3697,638],{"fence":397},[390,3699,3700],{"encoding":392},"\\log \\pi_{\\theta}(a|s) = -\\frac{1}{2}\\left(\\sum_{i=1}^k \\left(\\frac{(a_i - \\mu_i)^2}{\\sigma_i^2} + 2 \\log \\sigma_i \\right) + k \\log 2\\pi \\right)",[367,3702,3704,3785],{"className":3703,"ariaHidden":397},[396],[367,3705,3707,3710,3715,3718,3761,3764,3767,3770,3773,3776,3779,3782],{"className":3706},[401],[367,3708],{"className":3709,"style":726},[405],[367,3711,1765,3713],{"className":3712},[1653],[367,3714,1769],{"style":1768},[367,3716],{"className":3717,"style":1773},[711],[367,3719,3721,3724],{"className":3720},[410],[367,3722,795],{"className":3723,"style":811},[410,411],[367,3725,3727],{"className":3726},[661],[367,3728,3730,3753],{"className":3729},[665,666],[367,3731,3733,3750],{"className":3732},[670],[367,3734,3736],{"className":3735,"style":1353},[674],[367,3737,3738,3741],{"style":1465},[367,3739],{"className":3740,"style":683},[682],[367,3742,3744],{"className":3743},[687,688,689,690],[367,3745,3747],{"className":3746},[410,690],[367,3748,1004],{"className":3749,"style":1021},[410,411,690],[367,3751,698],{"className":3752},[697],[367,3754,3756],{"className":3755},[670],[367,3757,3759],{"className":3758,"style":705},[674],[367,3760],{},[367,3762,629],{"className":3763},[733],[367,3765,216],{"className":3766},[410,411],[367,3768,846],{"className":3769},[410],[367,3771,388],{"className":3772},[410,411],[367,3774,638],{"className":3775},[777],[367,3777],{"className":3778,"style":712},[711],[367,3780,623],{"className":3781},[716],[367,3783],{"className":3784,"style":712},[711],[367,3786,3788,3792,3795,3865,3868],{"className":3787},[401],[367,3789],{"className":3790,"style":3791},[405],"height:3.1138em;vertical-align:-1.2777em;",[367,3793,2650],{"className":3794},[410],[367,3796,3798,3802,3862],{"className":3797},[410],[367,3799],{"className":3800},[733,3801],"nulldelimiter",[367,3803,3805],{"className":3804},[3599],[367,3806,3808,3853],{"className":3807},[665,666],[367,3809,3811,3850],{"className":3810},[670],[367,3812,3815,3827,3838],{"className":3813,"style":3814},[674],"height:1.3214em;",[367,3816,3818,3821],{"style":3817},"top:-2.314em;",[367,3819],{"className":3820,"style":1197},[682],[367,3822,3824],{"className":3823},[410],[367,3825,3606],{"className":3826},[410],[367,3828,3830,3833],{"style":3829},"top:-3.23em;",[367,3831],{"className":3832,"style":1197},[682],[367,3834],{"className":3835,"style":3837},[3836],"frac-line","border-bottom-width:0.04em;",[367,3839,3841,3844],{"style":3840},"top:-3.677em;",[367,3842],{"className":3843,"style":1197},[682],[367,3845,3847],{"className":3846},[410],[367,3848,3603],{"className":3849},[410],[367,3851,698],{"className":3852},[697],[367,3854,3856],{"className":3855},[670],[367,3857,3860],{"className":3858,"style":3859},[674],"height:0.686em;",[367,3861],{},[367,3863],{"className":3864},[777,3801],[367,3866],{"className":3867,"style":1773},[711],[367,3869,3871,3879,3952,3955,4270,4273,4276,4279,4282,4285,4290,4293,4296,4299],{"className":3870},[2170],[367,3872,3874],{"className":3873,"style":2178},[733,2177],[367,3875,629],{"className":3876},[3877,3878],"delimsizing","size4",[367,3880,3883],{"className":3881},[1653,3882],"op-limits",[367,3884,3886,3943],{"className":3885},[665,666],[367,3887,3889,3940],{"className":3888},[670],[367,3890,3893,3915,3928],{"className":3891,"style":3892},[674],"height:1.8361em;",[367,3894,3896,3900],{"style":3895},"top:-1.8723em;margin-left:0em;",[367,3897],{"className":3898,"style":3899},[682],"height:3.05em;",[367,3901,3903],{"className":3902},[687,688,689,690],[367,3904,3906,3909,3912],{"className":3905},[410,690],[367,3907,3621],{"className":3908},[410,411,690],[367,3910,623],{"className":3911},[716,690],[367,3913,3603],{"className":3914},[410,690],[367,3916,3918,3921],{"style":3917},"top:-3.05em;",[367,3919],{"className":3920,"style":3899},[682],[367,3922,3923],{},[367,3924,3616],{"className":3925},[1653,3926,3927],"op-symbol","large-op",[367,3929,3931,3934],{"style":3930},"top:-4.3em;margin-left:0em;",[367,3932],{"className":3933,"style":3899},[682],[367,3935,3937],{"className":3936},[687,688,689,690],[367,3938,3513],{"className":3939,"style":3528},[410,411,690],[367,3941,698],{"className":3942},[697],[367,3944,3946],{"className":3945},[670],[367,3947,3950],{"className":3948,"style":3949},[674],"height:1.2777em;",[367,3951],{},[367,3953],{"className":3954,"style":1773},[711],[367,3956,3958,3964,4201,4204,4207,4210,4213,4216,4221,4224,4264],{"className":3957},[2170],[367,3959,3961],{"className":3960,"style":2178},[733,2177],[367,3962,629],{"className":3963},[3877,689],[367,3965,3967,3970,4198],{"className":3966},[410],[367,3968],{"className":3969},[733,3801],[367,3971,3973],{"className":3972},[3599],[367,3974,3976,4189],{"className":3975},[665,666],[367,3977,3979,4186],{"className":3978},[670],[367,3980,3983,4046,4054],{"className":3981,"style":3982},[674],"height:1.4911em;",[367,3984,3985,3988],{"style":3817},[367,3986],{"className":3987,"style":1197},[682],[367,3989,3991],{"className":3990},[410],[367,3992,3994,3997],{"className":3993},[410],[367,3995,2454],{"className":3996,"style":811},[410,411],[367,3998,4000],{"className":3999},[661],[367,4001,4003,4037],{"className":4002},[665,666],[367,4004,4006,4034],{"className":4005},[670],[367,4007,4010,4022],{"className":4008,"style":4009},[674],"height:0.7959em;",[367,4011,4013,4016],{"style":4012},"top:-2.4231em;margin-left:-0.0359em;margin-right:0.05em;",[367,4014],{"className":4015,"style":683},[682],[367,4017,4019],{"className":4018},[687,688,689,690],[367,4020,3621],{"className":4021},[410,411,690],[367,4023,4025,4028],{"style":4024},"top:-3.0448em;margin-right:0.05em;",[367,4026],{"className":4027,"style":683},[682],[367,4029,4031],{"className":4030},[687,688,689,690],[367,4032,3606],{"className":4033},[410,690],[367,4035,698],{"className":4036},[697],[367,4038,4040],{"className":4039},[670],[367,4041,4044],{"className":4042,"style":4043},[674],"height:0.2769em;",[367,4045],{},[367,4047,4048,4051],{"style":3829},[367,4049],{"className":4050,"style":1197},[682],[367,4052],{"className":4053,"style":3837},[3836],[367,4055,4056,4059],{"style":3840},[367,4057],{"className":4058,"style":1197},[682],[367,4060,4062,4065,4106,4109,4112,4115,4155],{"className":4061},[410],[367,4063,629],{"className":4064},[733],[367,4066,4068,4071],{"className":4067},[410],[367,4069,216],{"className":4070},[410,411],[367,4072,4074],{"className":4073},[661],[367,4075,4077,4098],{"className":4076},[665,666],[367,4078,4080,4095],{"className":4079},[670],[367,4081,4084],{"className":4082,"style":4083},[674],"height:0.3117em;",[367,4085,4086,4089],{"style":678},[367,4087],{"className":4088,"style":683},[682],[367,4090,4092],{"className":4091},[687,688,689,690],[367,4093,3621],{"className":4094},[410,411,690],[367,4096,698],{"className":4097},[697],[367,4099,4101],{"className":4100},[670],[367,4102,4104],{"className":4103,"style":705},[674],[367,4105],{},[367,4107],{"className":4108,"style":3132},[711],[367,4110,2650],{"className":4111},[3136],[367,4113],{"className":4114,"style":3132},[711],[367,4116,4118,4121],{"className":4117},[410],[367,4119,577],{"className":4120},[410,411],[367,4122,4124],{"className":4123},[661],[367,4125,4127,4147],{"className":4126},[665,666],[367,4128,4130,4144],{"className":4129},[670],[367,4131,4133],{"className":4132,"style":4083},[674],[367,4134,4135,4138],{"style":678},[367,4136],{"className":4137,"style":683},[682],[367,4139,4141],{"className":4140},[687,688,689,690],[367,4142,3621],{"className":4143},[410,411,690],[367,4145,698],{"className":4146},[697],[367,4148,4150],{"className":4149},[670],[367,4151,4153],{"className":4152,"style":705},[674],[367,4154],{},[367,4156,4158,4161],{"className":4157},[777],[367,4159,638],{"className":4160},[777],[367,4162,4164],{"className":4163},[661],[367,4165,4167],{"className":4166},[665],[367,4168,4170],{"className":4169},[670],[367,4171,4174],{"className":4172,"style":4173},[674],"height:0.8141em;",[367,4175,4177,4180],{"style":4176},"top:-3.063em;margin-right:0.05em;",[367,4178],{"className":4179,"style":683},[682],[367,4181,4183],{"className":4182},[687,688,689,690],[367,4184,3606],{"className":4185},[410,690],[367,4187,698],{"className":4188},[697],[367,4190,4192],{"className":4191},[670],[367,4193,4196],{"className":4194,"style":4195},[674],"height:0.9629em;",[367,4197],{},[367,4199],{"className":4200},[777,3801],[367,4202],{"className":4203,"style":3132},[711],[367,4205,3029],{"className":4206},[3136],[367,4208],{"className":4209,"style":3132},[711],[367,4211,3606],{"className":4212},[410],[367,4214],{"className":4215,"style":1773},[711],[367,4217,1765,4219],{"className":4218},[1653],[367,4220,1769],{"style":1768},[367,4222],{"className":4223,"style":1773},[711],[367,4225,4227,4230],{"className":4226},[410],[367,4228,2454],{"className":4229,"style":811},[410,411],[367,4231,4233],{"className":4232},[661],[367,4234,4236,4256],{"className":4235},[665,666],[367,4237,4239,4253],{"className":4238},[670],[367,4240,4242],{"className":4241,"style":4083},[674],[367,4243,4244,4247],{"style":1465},[367,4245],{"className":4246,"style":683},[682],[367,4248,4250],{"className":4249},[687,688,689,690],[367,4251,3621],{"className":4252},[410,411,690],[367,4254,698],{"className":4255},[697],[367,4257,4259],{"className":4258},[670],[367,4260,4262],{"className":4261,"style":705},[674],[367,4263],{},[367,4265,4267],{"className":4266,"style":2178},[777,2177],[367,4268,638],{"className":4269},[3877,689],[367,4271],{"className":4272,"style":3132},[711],[367,4274,3029],{"className":4275},[3136],[367,4277],{"className":4278,"style":3132},[711],[367,4280,3513],{"className":4281,"style":3528},[410,411],[367,4283],{"className":4284,"style":1773},[711],[367,4286,1765,4288],{"className":4287},[1653],[367,4289,1769],{"style":1768},[367,4291],{"className":4292,"style":1773},[711],[367,4294,3606],{"className":4295},[410],[367,4297,795],{"className":4298,"style":811},[410,411],[367,4300,4302],{"className":4301,"style":2178},[777,2177],[367,4303,638],{"className":4304},[3877,3878],[357,4306,4307],{"id":4307},"轨迹",[212,4309,4310,4311,4342],{},"轨迹（trajectory）",[367,4312,4314,4329],{"className":4313},[370],[367,4315,4317],{"className":4316},[374],[376,4318,4319],{"xmlns":378},[380,4320,4321,4326],{},[383,4322,4323],{},[386,4324,4325],{},"τ",[390,4327,4328],{"encoding":392},"\\tau",[367,4330,4332],{"className":4331,"ariaHidden":397},[396],[367,4333,4335,4338],{"className":4334},[401],[367,4336],{"className":4337,"style":406},[405],[367,4339,4325],{"className":4340,"style":4341},[410,411],"margin-right:0.1132em;"," 是世界中状态与动作构成的序列：",[367,4344,4346],{"className":4345},[598],[367,4347,4349,4408],{"className":4348},[370],[367,4350,4352],{"className":4351},[374],[376,4353,4354],{"xmlns":378,"display":607},[380,4355,4356,4405],{},[383,4357,4358,4360,4362,4364,4370,4372,4378,4380,4386,4388,4394,4396,4399,4401,4403],{},[386,4359,4325],{},[621,4361,623],{},[621,4363,629],{"stretchy":628},[613,4365,4366,4368],{},[386,4367,388],{},[2932,4369,1083],{},[621,4371,2656],{"separator":397},[613,4373,4374,4376],{},[386,4375,216],{},[2932,4377,1083],{},[621,4379,2656],{"separator":397},[613,4381,4382,4384],{},[386,4383,388],{},[2932,4385,3603],{},[621,4387,2656],{"separator":397},[613,4389,4390,4392],{},[386,4391,216],{},[2932,4393,3603],{},[621,4395,2656],{"separator":397},[386,4397,4398],{"mathvariant":845},".",[386,4400,4398],{"mathvariant":845},[386,4402,4398],{"mathvariant":845},[621,4404,638],{"stretchy":628},[390,4406,4407],{"encoding":392},"\\tau = (s_0, a_0, s_1, a_1, ...)",[367,4409,4411,4429],{"className":4410,"ariaHidden":397},[396],[367,4412,4414,4417,4420,4423,4426],{"className":4413},[401],[367,4415],{"className":4416,"style":406},[405],[367,4418,4325],{"className":4419,"style":4341},[410,411],[367,4421],{"className":4422,"style":712},[711],[367,4424,623],{"className":4425},[716],[367,4427],{"className":4428,"style":712},[711],[367,4430,4432,4435,4438,4479,4482,4485,4525,4528,4531,4571,4574,4577,4617,4620,4623,4627],{"className":4431},[401],[367,4433],{"className":4434,"style":726},[405],[367,4436,629],{"className":4437},[733],[367,4439,4441,4444],{"className":4440},[410],[367,4442,388],{"className":4443},[410,411],[367,4445,4447],{"className":4446},[661],[367,4448,4450,4471],{"className":4449},[665,666],[367,4451,4453,4468],{"className":4452},[670],[367,4454,4457],{"className":4455,"style":4456},[674],"height:0.3011em;",[367,4458,4459,4462],{"style":678},[367,4460],{"className":4461,"style":683},[682],[367,4463,4465],{"className":4464},[687,688,689,690],[367,4466,1083],{"className":4467},[410,690],[367,4469,698],{"className":4470},[697],[367,4472,4474],{"className":4473},[670],[367,4475,4477],{"className":4476,"style":705},[674],[367,4478],{},[367,4480,2656],{"className":4481},[2685],[367,4483],{"className":4484,"style":1773},[711],[367,4486,4488,4491],{"className":4487},[410],[367,4489,216],{"className":4490},[410,411],[367,4492,4494],{"className":4493},[661],[367,4495,4497,4517],{"className":4496},[665,666],[367,4498,4500,4514],{"className":4499},[670],[367,4501,4503],{"className":4502,"style":4456},[674],[367,4504,4505,4508],{"style":678},[367,4506],{"className":4507,"style":683},[682],[367,4509,4511],{"className":4510},[687,688,689,690],[367,4512,1083],{"className":4513},[410,690],[367,4515,698],{"className":4516},[697],[367,4518,4520],{"className":4519},[670],[367,4521,4523],{"className":4522,"style":705},[674],[367,4524],{},[367,4526,2656],{"className":4527},[2685],[367,4529],{"className":4530,"style":1773},[711],[367,4532,4534,4537],{"className":4533},[410],[367,4535,388],{"className":4536},[410,411],[367,4538,4540],{"className":4539},[661],[367,4541,4543,4563],{"className":4542},[665,666],[367,4544,4546,4560],{"className":4545},[670],[367,4547,4549],{"className":4548,"style":4456},[674],[367,4550,4551,4554],{"style":678},[367,4552],{"className":4553,"style":683},[682],[367,4555,4557],{"className":4556},[687,688,689,690],[367,4558,3603],{"className":4559},[410,690],[367,4561,698],{"className":4562},[697],[367,4564,4566],{"className":4565},[670],[367,4567,4569],{"className":4568,"style":705},[674],[367,4570],{},[367,4572,2656],{"className":4573},[2685],[367,4575],{"className":4576,"style":1773},[711],[367,4578,4580,4583],{"className":4579},[410],[367,4581,216],{"className":4582},[410,411],[367,4584,4586],{"className":4585},[661],[367,4587,4589,4609],{"className":4588},[665,666],[367,4590,4592,4606],{"className":4591},[670],[367,4593,4595],{"className":4594,"style":4456},[674],[367,4596,4597,4600],{"style":678},[367,4598],{"className":4599,"style":683},[682],[367,4601,4603],{"className":4602},[687,688,689,690],[367,4604,3603],{"className":4605},[410,690],[367,4607,698],{"className":4608},[697],[367,4610,4612],{"className":4611},[670],[367,4613,4615],{"className":4614,"style":705},[674],[367,4616],{},[367,4618,2656],{"className":4619},[2685],[367,4621],{"className":4622,"style":1773},[711],[367,4624,4626],{"className":4625},[410],"...",[367,4628,638],{"className":4629},[777],[212,4631,4632,4633,4703,4704,4707,4708,594],{},"世界的第一个状态 ",[367,4634,4636,4654],{"className":4635},[370],[367,4637,4639],{"className":4638},[374],[376,4640,4641],{"xmlns":378},[380,4642,4643,4651],{},[383,4644,4645],{},[613,4646,4647,4649],{},[386,4648,388],{},[2932,4650,1083],{},[390,4652,4653],{"encoding":392},"s_0",[367,4655,4657],{"className":4656,"ariaHidden":397},[396],[367,4658,4660,4663],{"className":4659},[401],[367,4661],{"className":4662,"style":651},[405],[367,4664,4666,4669],{"className":4665},[410],[367,4667,388],{"className":4668},[410,411],[367,4670,4672],{"className":4671},[661],[367,4673,4675,4695],{"className":4674},[665,666],[367,4676,4678,4692],{"className":4677},[670],[367,4679,4681],{"className":4680,"style":4456},[674],[367,4682,4683,4686],{"style":678},[367,4684],{"className":4685,"style":683},[682],[367,4687,4689],{"className":4688},[687,688,689,690],[367,4690,1083],{"className":4691},[410,690],[367,4693,698],{"className":4694},[697],[367,4696,4698],{"className":4697},[670],[367,4699,4701],{"className":4700,"style":705},[674],[367,4702],{}," 从",[315,4705,4706],{},"初始状态分布（start-state distribution）"," 中随机采样，后者有时记作 ",[367,4709,4711,4730],{"className":4710},[370],[367,4712,4714],{"className":4713},[374],[376,4715,4716],{"xmlns":378},[380,4717,4718,4727],{},[383,4719,4720],{},[613,4721,4722,4725],{},[386,4723,4724],{},"ρ",[2932,4726,1083],{},[390,4728,4729],{"encoding":392},"\\rho_0",[367,4731,4733],{"className":4732,"ariaHidden":397},[396],[367,4734,4736,4739],{"className":4735},[401],[367,4737],{"className":4738,"style":590},[405],[367,4740,4742,4745],{"className":4741},[410],[367,4743,4724],{"className":4744},[410,411],[367,4746,4748],{"className":4747},[661],[367,4749,4751,4771],{"className":4750},[665,666],[367,4752,4754,4768],{"className":4753},[670],[367,4755,4757],{"className":4756,"style":4456},[674],[367,4758,4759,4762],{"style":678},[367,4760],{"className":4761,"style":683},[682],[367,4763,4765],{"className":4764},[687,688,689,690],[367,4766,1083],{"className":4767},[410,690],[367,4769,698],{"className":4770},[697],[367,4772,4774],{"className":4773},[670],[367,4775,4777],{"className":4776,"style":705},[674],[367,4778],{},[367,4780,4782],{"className":4781},[598],[367,4783,4785,4817],{"className":4784},[370],[367,4786,4788],{"className":4787},[374],[376,4789,4790],{"xmlns":378,"display":607},[380,4791,4792,4814],{},[383,4793,4794,4800,4802,4808,4810,4812],{},[613,4795,4796,4798],{},[386,4797,388],{},[2932,4799,1083],{},[621,4801,835],{},[613,4803,4804,4806],{},[386,4805,4724],{},[2932,4807,1083],{},[621,4809,629],{"stretchy":628},[621,4811,842],{},[621,4813,638],{"stretchy":628},[390,4815,4816],{"encoding":392},"s_0 \\sim \\rho_0(\\cdot)",[367,4818,4820,4875],{"className":4819,"ariaHidden":397},[396],[367,4821,4823,4826,4866,4869,4872],{"className":4822},[401],[367,4824],{"className":4825,"style":651},[405],[367,4827,4829,4832],{"className":4828},[410],[367,4830,388],{"className":4831},[410,411],[367,4833,4835],{"className":4834},[661],[367,4836,4838,4858],{"className":4837},[665,666],[367,4839,4841,4855],{"className":4840},[670],[367,4842,4844],{"className":4843,"style":4456},[674],[367,4845,4846,4849],{"style":678},[367,4847],{"className":4848,"style":683},[682],[367,4850,4852],{"className":4851},[687,688,689,690],[367,4853,1083],{"className":4854},[410,690],[367,4856,698],{"className":4857},[697],[367,4859,4861],{"className":4860},[670],[367,4862,4864],{"className":4863,"style":705},[674],[367,4865],{},[367,4867],{"className":4868,"style":712},[711],[367,4870,835],{"className":4871},[716],[367,4873],{"className":4874,"style":712},[711],[367,4876,4878,4881,4921,4924,4927],{"className":4877},[401],[367,4879],{"className":4880,"style":726},[405],[367,4882,4884,4887],{"className":4883},[410],[367,4885,4724],{"className":4886},[410,411],[367,4888,4890],{"className":4889},[661],[367,4891,4893,4913],{"className":4892},[665,666],[367,4894,4896,4910],{"className":4895},[670],[367,4897,4899],{"className":4898,"style":4456},[674],[367,4900,4901,4904],{"style":678},[367,4902],{"className":4903,"style":683},[682],[367,4905,4907],{"className":4906},[687,688,689,690],[367,4908,1083],{"className":4909},[410,690],[367,4911,698],{"className":4912},[697],[367,4914,4916],{"className":4915},[670],[367,4917,4919],{"className":4918,"style":705},[674],[367,4920],{},[367,4922,629],{"className":4923},[733],[367,4925,842],{"className":4926},[410],[367,4928,638],{"className":4929},[777],[212,4931,4932,4933,4962,4963,5033,5034,5087,5088,5175,5176,5246],{},"状态转移（从时刻 ",[367,4934,4936,4949],{"className":4935},[370],[367,4937,4939],{"className":4938},[374],[376,4940,4941],{"xmlns":378},[380,4942,4943,4947],{},[383,4944,4945],{},[386,4946,619],{},[390,4948,619],{"encoding":392},[367,4950,4952],{"className":4951,"ariaHidden":397},[396],[367,4953,4955,4959],{"className":4954},[401],[367,4956],{"className":4957,"style":4958},[405],"height:0.6151em;",[367,4960,619],{"className":4961},[410,411]," 的世界状态 ",[367,4964,4966,4984],{"className":4965},[370],[367,4967,4969],{"className":4968},[374],[376,4970,4971],{"xmlns":378},[380,4972,4973,4981],{},[383,4974,4975],{},[613,4976,4977,4979],{},[386,4978,388],{},[386,4980,619],{},[390,4982,4983],{"encoding":392},"s_t",[367,4985,4987],{"className":4986,"ariaHidden":397},[396],[367,4988,4990,4993],{"className":4989},[401],[367,4991],{"className":4992,"style":651},[405],[367,4994,4996,4999],{"className":4995},[410],[367,4997,388],{"className":4998},[410,411],[367,5000,5002],{"className":5001},[661],[367,5003,5005,5025],{"className":5004},[665,666],[367,5006,5008,5022],{"className":5007},[670],[367,5009,5011],{"className":5010,"style":675},[674],[367,5012,5013,5016],{"style":678},[367,5014],{"className":5015,"style":683},[682],[367,5017,5019],{"className":5018},[687,688,689,690],[367,5020,619],{"className":5021},[410,411,690],[367,5023,698],{"className":5024},[697],[367,5026,5028],{"className":5027},[670],[367,5029,5031],{"className":5030,"style":705},[674],[367,5032],{}," 到时刻 ",[367,5035,5037,5055],{"className":5036},[370],[367,5038,5040],{"className":5039},[374],[376,5041,5042],{"xmlns":378},[380,5043,5044,5052],{},[383,5045,5046,5048,5050],{},[386,5047,619],{},[621,5049,3029],{},[2932,5051,3603],{},[390,5053,5054],{"encoding":392},"t+1",[367,5056,5058,5077],{"className":5057,"ariaHidden":397},[396],[367,5059,5061,5065,5068,5071,5074],{"className":5060},[401],[367,5062],{"className":5063,"style":5064},[405],"height:0.6984em;vertical-align:-0.0833em;",[367,5066,619],{"className":5067},[410,411],[367,5069],{"className":5070,"style":3132},[711],[367,5072,3029],{"className":5073},[3136],[367,5075],{"className":5076,"style":3132},[711],[367,5078,5080,5084],{"className":5079},[401],[367,5081],{"className":5082,"style":5083},[405],"height:0.6444em;",[367,5085,3603],{"className":5086},[410]," 的状态 ",[367,5089,5091,5115],{"className":5090},[370],[367,5092,5094],{"className":5093},[374],[376,5095,5096],{"xmlns":378},[380,5097,5098,5112],{},[383,5099,5100],{},[613,5101,5102,5104],{},[386,5103,388],{},[383,5105,5106,5108,5110],{},[386,5107,619],{},[621,5109,3029],{},[2932,5111,3603],{},[390,5113,5114],{"encoding":392},"s_{t+1}",[367,5116,5118],{"className":5117,"ariaHidden":397},[396],[367,5119,5121,5125],{"className":5120},[401],[367,5122],{"className":5123,"style":5124},[405],"height:0.6389em;vertical-align:-0.2083em;",[367,5126,5128,5131],{"className":5127},[410],[367,5129,388],{"className":5130},[410,411],[367,5132,5134],{"className":5133},[661],[367,5135,5137,5166],{"className":5136},[665,666],[367,5138,5140,5163],{"className":5139},[670],[367,5141,5143],{"className":5142,"style":4456},[674],[367,5144,5145,5148],{"style":678},[367,5146],{"className":5147,"style":683},[682],[367,5149,5151],{"className":5150},[687,688,689,690],[367,5152,5154,5157,5160],{"className":5153},[410,690],[367,5155,619],{"className":5156},[410,411,690],[367,5158,3029],{"className":5159},[3136,690],[367,5161,3603],{"className":5162},[410,690],[367,5164,698],{"className":5165},[697],[367,5167,5169],{"className":5168},[670],[367,5170,5173],{"className":5171,"style":5172},[674],"height:0.2083em;",[367,5174],{}," 之间世界发生了什么）由环境的自然法则支配，且只依赖最近的动作 ",[367,5177,5179,5197],{"className":5178},[370],[367,5180,5182],{"className":5181},[374],[376,5183,5184],{"xmlns":378},[380,5185,5186,5194],{},[383,5187,5188],{},[613,5189,5190,5192],{},[386,5191,216],{},[386,5193,619],{},[390,5195,5196],{"encoding":392},"a_t",[367,5198,5200],{"className":5199,"ariaHidden":397},[396],[367,5201,5203,5206],{"className":5202},[401],[367,5204],{"className":5205,"style":651},[405],[367,5207,5209,5212],{"className":5208},[410],[367,5210,216],{"className":5211},[410,411],[367,5213,5215],{"className":5214},[661],[367,5216,5218,5238],{"className":5217},[665,666],[367,5219,5221,5235],{"className":5220},[670],[367,5222,5224],{"className":5223,"style":675},[674],[367,5225,5226,5229],{"style":678},[367,5227],{"className":5228,"style":683},[682],[367,5230,5232],{"className":5231},[687,688,689,690],[367,5233,619],{"className":5234},[410,411,690],[367,5236,698],{"className":5237},[697],[367,5239,5241],{"className":5240},[670],[367,5242,5244],{"className":5243,"style":705},[674],[367,5245],{},"。它可以是确定型的：",[367,5248,5250],{"className":5249},[598],[367,5251,5253,5300],{"className":5252},[370],[367,5254,5256],{"className":5255},[374],[376,5257,5258],{"xmlns":378,"display":607},[380,5259,5260,5297],{},[383,5261,5262,5274,5276,5279,5281,5287,5289,5295],{},[613,5263,5264,5266],{},[386,5265,388],{},[383,5267,5268,5270,5272],{},[386,5269,619],{},[621,5271,3029],{},[2932,5273,3603],{},[621,5275,623],{},[386,5277,5278],{},"f",[621,5280,629],{"stretchy":628},[613,5282,5283,5285],{},[386,5284,388],{},[386,5286,619],{},[621,5288,2656],{"separator":397},[613,5290,5291,5293],{},[386,5292,216],{},[386,5294,619],{},[621,5296,638],{"stretchy":628},[390,5298,5299],{"encoding":392},"s_{t+1} = f(s_t, a_t)",[367,5301,5303,5367],{"className":5302,"ariaHidden":397},[396],[367,5304,5306,5309,5358,5361,5364],{"className":5305},[401],[367,5307],{"className":5308,"style":5124},[405],[367,5310,5312,5315],{"className":5311},[410],[367,5313,388],{"className":5314},[410,411],[367,5316,5318],{"className":5317},[661],[367,5319,5321,5350],{"className":5320},[665,666],[367,5322,5324,5347],{"className":5323},[670],[367,5325,5327],{"className":5326,"style":4456},[674],[367,5328,5329,5332],{"style":678},[367,5330],{"className":5331,"style":683},[682],[367,5333,5335],{"className":5334},[687,688,689,690],[367,5336,5338,5341,5344],{"className":5337},[410,690],[367,5339,619],{"className":5340},[410,411,690],[367,5342,3029],{"className":5343},[3136,690],[367,5345,3603],{"className":5346},[410,690],[367,5348,698],{"className":5349},[697],[367,5351,5353],{"className":5352},[670],[367,5354,5356],{"className":5355,"style":5172},[674],[367,5357],{},[367,5359],{"className":5360,"style":712},[711],[367,5362,623],{"className":5363},[716],[367,5365],{"className":5366,"style":712},[711],[367,5368,5370,5373,5377,5380,5420,5423,5426,5466],{"className":5369},[401],[367,5371],{"className":5372,"style":726},[405],[367,5374,5278],{"className":5375,"style":5376},[410,411],"margin-right:0.1076em;",[367,5378,629],{"className":5379},[733],[367,5381,5383,5386],{"className":5382},[410],[367,5384,388],{"className":5385},[410,411],[367,5387,5389],{"className":5388},[661],[367,5390,5392,5412],{"className":5391},[665,666],[367,5393,5395,5409],{"className":5394},[670],[367,5396,5398],{"className":5397,"style":675},[674],[367,5399,5400,5403],{"style":678},[367,5401],{"className":5402,"style":683},[682],[367,5404,5406],{"className":5405},[687,688,689,690],[367,5407,619],{"className":5408},[410,411,690],[367,5410,698],{"className":5411},[697],[367,5413,5415],{"className":5414},[670],[367,5416,5418],{"className":5417,"style":705},[674],[367,5419],{},[367,5421,2656],{"className":5422},[2685],[367,5424],{"className":5425,"style":1773},[711],[367,5427,5429,5432],{"className":5428},[410],[367,5430,216],{"className":5431},[410,411],[367,5433,5435],{"className":5434},[661],[367,5436,5438,5458],{"className":5437},[665,666],[367,5439,5441,5455],{"className":5440},[670],[367,5442,5444],{"className":5443,"style":675},[674],[367,5445,5446,5449],{"style":678},[367,5447],{"className":5448,"style":683},[682],[367,5450,5452],{"className":5451},[687,688,689,690],[367,5453,619],{"className":5454},[410,411,690],[367,5456,698],{"className":5457},[697],[367,5459,5461],{"className":5460},[670],[367,5462,5464],{"className":5463,"style":705},[674],[367,5465],{},[367,5467,638],{"className":5468},[777],[212,5470,5471],{},"也可以是随机型的：",[367,5473,5475],{"className":5474},[598],[367,5476,5478,5528],{"className":5477},[370],[367,5479,5481],{"className":5480},[374],[376,5482,5483],{"xmlns":378,"display":607},[380,5484,5485,5525],{},[383,5486,5487,5499,5501,5503,5505,5507,5509,5515,5517,5523],{},[613,5488,5489,5491],{},[386,5490,388],{},[383,5492,5493,5495,5497],{},[386,5494,619],{},[621,5496,3029],{},[2932,5498,3603],{},[621,5500,835],{},[386,5502,1895],{},[621,5504,629],{"stretchy":628},[621,5506,842],{},[386,5508,846],{"mathvariant":845},[613,5510,5511,5513],{},[386,5512,388],{},[386,5514,619],{},[621,5516,2656],{"separator":397},[613,5518,5519,5521],{},[386,5520,216],{},[386,5522,619],{},[621,5524,638],{"stretchy":628},[390,5526,5527],{"encoding":392},"s_{t+1} \\sim P(\\cdot|s_t, a_t)",[367,5529,5531,5595],{"className":5530,"ariaHidden":397},[396],[367,5532,5534,5537,5586,5589,5592],{"className":5533},[401],[367,5535],{"className":5536,"style":5124},[405],[367,5538,5540,5543],{"className":5539},[410],[367,5541,388],{"className":5542},[410,411],[367,5544,5546],{"className":5545},[661],[367,5547,5549,5578],{"className":5548},[665,666],[367,5550,5552,5575],{"className":5551},[670],[367,5553,5555],{"className":5554,"style":4456},[674],[367,5556,5557,5560],{"style":678},[367,5558],{"className":5559,"style":683},[682],[367,5561,5563],{"className":5562},[687,688,689,690],[367,5564,5566,5569,5572],{"className":5565},[410,690],[367,5567,619],{"className":5568},[410,411,690],[367,5570,3029],{"className":5571},[3136,690],[367,5573,3603],{"className":5574},[410,690],[367,5576,698],{"className":5577},[697],[367,5579,5581],{"className":5580},[670],[367,5582,5584],{"className":5583,"style":5172},[674],[367,5585],{},[367,5587],{"className":5588,"style":712},[711],[367,5590,835],{"className":5591},[716],[367,5593],{"className":5594,"style":712},[711],[367,5596,5598,5601,5604,5607,5610,5613,5653,5656,5659,5699],{"className":5597},[401],[367,5599],{"className":5600,"style":726},[405],[367,5602,1895],{"className":5603,"style":1922},[410,411],[367,5605,629],{"className":5606},[733],[367,5608,842],{"className":5609},[410],[367,5611,846],{"className":5612},[410],[367,5614,5616,5619],{"className":5615},[410],[367,5617,388],{"className":5618},[410,411],[367,5620,5622],{"className":5621},[661],[367,5623,5625,5645],{"className":5624},[665,666],[367,5626,5628,5642],{"className":5627},[670],[367,5629,5631],{"className":5630,"style":675},[674],[367,5632,5633,5636],{"style":678},[367,5634],{"className":5635,"style":683},[682],[367,5637,5639],{"className":5638},[687,688,689,690],[367,5640,619],{"className":5641},[410,411,690],[367,5643,698],{"className":5644},[697],[367,5646,5648],{"className":5647},[670],[367,5649,5651],{"className":5650,"style":705},[674],[367,5652],{},[367,5654,2656],{"className":5655},[2685],[367,5657],{"className":5658,"style":1773},[711],[367,5660,5662,5665],{"className":5661},[410],[367,5663,216],{"className":5664},[410,411],[367,5666,5668],{"className":5667},[661],[367,5669,5671,5691],{"className":5670},[665,666],[367,5672,5674,5688],{"className":5673},[670],[367,5675,5677],{"className":5676,"style":675},[674],[367,5678,5679,5682],{"style":678},[367,5680],{"className":5681,"style":683},[682],[367,5683,5685],{"className":5684},[687,688,689,690],[367,5686,619],{"className":5687},[410,411,690],[367,5689,698],{"className":5690},[697],[367,5692,5694],{"className":5693},[670],[367,5695,5697],{"className":5696,"style":705},[674],[367,5698],{},[367,5700,638],{"className":5701},[777],[212,5703,5704],{},"动作则由智能体依照其策略产生。",[208,5706,5707],{"title":462},[212,5708,5709,5710,5713],{},"轨迹也常被称为",[315,5711,5712],{},"回合（episode）"," 或 rollout。",[357,5715,5716],{"id":5716},"奖励与回报",[212,5718,5719,5720,5750],{},"奖励函数 ",[367,5721,5723,5737],{"className":5722},[370],[367,5724,5726],{"className":5725},[374],[376,5727,5728],{"xmlns":378},[380,5729,5730,5735],{},[383,5731,5732],{},[386,5733,5734],{},"R",[390,5736,5734],{"encoding":392},[367,5738,5740],{"className":5739,"ariaHidden":397},[396],[367,5741,5743,5746],{"className":5742},[401],[367,5744],{"className":5745,"style":2333},[405],[367,5747,5734],{"className":5748,"style":5749},[410,411],"margin-right:0.0077em;"," 在强化学习中至关重要。它依赖当前的世界状态、刚采取的动作和下一个世界状态：",[367,5752,5754],{"className":5753},[598],[367,5755,5757,5812],{"className":5756},[370],[367,5758,5760],{"className":5759},[374],[376,5761,5762],{"xmlns":378,"display":607},[380,5763,5764,5809],{},[383,5765,5766,5773,5775,5777,5779,5785,5787,5793,5795,5807],{},[613,5767,5768,5771],{},[386,5769,5770],{},"r",[386,5772,619],{},[621,5774,623],{},[386,5776,5734],{},[621,5778,629],{"stretchy":628},[613,5780,5781,5783],{},[386,5782,388],{},[386,5784,619],{},[621,5786,2656],{"separator":397},[613,5788,5789,5791],{},[386,5790,216],{},[386,5792,619],{},[621,5794,2656],{"separator":397},[613,5796,5797,5799],{},[386,5798,388],{},[383,5800,5801,5803,5805],{},[386,5802,619],{},[621,5804,3029],{},[2932,5806,3603],{},[621,5808,638],{"stretchy":628},[390,5810,5811],{"encoding":392},"r_t = R(s_t, a_t, s_{t+1})",[367,5813,5815,5871],{"className":5814,"ariaHidden":397},[396],[367,5816,5818,5821,5862,5865,5868],{"className":5817},[401],[367,5819],{"className":5820,"style":651},[405],[367,5822,5824,5827],{"className":5823},[410],[367,5825,5770],{"className":5826,"style":1021},[410,411],[367,5828,5830],{"className":5829},[661],[367,5831,5833,5854],{"className":5832},[665,666],[367,5834,5836,5851],{"className":5835},[670],[367,5837,5839],{"className":5838,"style":675},[674],[367,5840,5842,5845],{"style":5841},"top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;",[367,5843],{"className":5844,"style":683},[682],[367,5846,5848],{"className":5847},[687,688,689,690],[367,5849,619],{"className":5850},[410,411,690],[367,5852,698],{"className":5853},[697],[367,5855,5857],{"className":5856},[670],[367,5858,5860],{"className":5859,"style":705},[674],[367,5861],{},[367,5863],{"className":5864,"style":712},[711],[367,5866,623],{"className":5867},[716],[367,5869],{"className":5870,"style":712},[711],[367,5872,5874,5877,5880,5883,5923,5926,5929,5969,5972,5975,6024],{"className":5873},[401],[367,5875],{"className":5876,"style":726},[405],[367,5878,5734],{"className":5879,"style":5749},[410,411],[367,5881,629],{"className":5882},[733],[367,5884,5886,5889],{"className":5885},[410],[367,5887,388],{"className":5888},[410,411],[367,5890,5892],{"className":5891},[661],[367,5893,5895,5915],{"className":5894},[665,666],[367,5896,5898,5912],{"className":5897},[670],[367,5899,5901],{"className":5900,"style":675},[674],[367,5902,5903,5906],{"style":678},[367,5904],{"className":5905,"style":683},[682],[367,5907,5909],{"className":5908},[687,688,689,690],[367,5910,619],{"className":5911},[410,411,690],[367,5913,698],{"className":5914},[697],[367,5916,5918],{"className":5917},[670],[367,5919,5921],{"className":5920,"style":705},[674],[367,5922],{},[367,5924,2656],{"className":5925},[2685],[367,5927],{"className":5928,"style":1773},[711],[367,5930,5932,5935],{"className":5931},[410],[367,5933,216],{"className":5934},[410,411],[367,5936,5938],{"className":5937},[661],[367,5939,5941,5961],{"className":5940},[665,666],[367,5942,5944,5958],{"className":5943},[670],[367,5945,5947],{"className":5946,"style":675},[674],[367,5948,5949,5952],{"style":678},[367,5950],{"className":5951,"style":683},[682],[367,5953,5955],{"className":5954},[687,688,689,690],[367,5956,619],{"className":5957},[410,411,690],[367,5959,698],{"className":5960},[697],[367,5962,5964],{"className":5963},[670],[367,5965,5967],{"className":5966,"style":705},[674],[367,5968],{},[367,5970,2656],{"className":5971},[2685],[367,5973],{"className":5974,"style":1773},[711],[367,5976,5978,5981],{"className":5977},[410],[367,5979,388],{"className":5980},[410,411],[367,5982,5984],{"className":5983},[661],[367,5985,5987,6016],{"className":5986},[665,666],[367,5988,5990,6013],{"className":5989},[670],[367,5991,5993],{"className":5992,"style":4456},[674],[367,5994,5995,5998],{"style":678},[367,5996],{"className":5997,"style":683},[682],[367,5999,6001],{"className":6000},[687,688,689,690],[367,6002,6004,6007,6010],{"className":6003},[410,690],[367,6005,619],{"className":6006},[410,411,690],[367,6008,3029],{"className":6009},[3136,690],[367,6011,3603],{"className":6012},[410,690],[367,6014,698],{"className":6015},[697],[367,6017,6019],{"className":6018},[670],[367,6020,6022],{"className":6021,"style":5172},[674],[367,6023],{},[367,6025,638],{"className":6026},[777],[212,6028,6029,6030,6178,6179,272],{},"不过它经常被简化成只依赖当前状态 ",[367,6031,6033,6065],{"className":6032},[370],[367,6034,6036],{"className":6035},[374],[376,6037,6038],{"xmlns":378},[380,6039,6040,6062],{},[383,6041,6042,6048,6050,6052,6054,6060],{},[613,6043,6044,6046],{},[386,6045,5770],{},[386,6047,619],{},[621,6049,623],{},[386,6051,5734],{},[621,6053,629],{"stretchy":628},[613,6055,6056,6058],{},[386,6057,388],{},[386,6059,619],{},[621,6061,638],{"stretchy":628},[390,6063,6064],{"encoding":392},"r_t = R(s_t)",[367,6066,6068,6123],{"className":6067,"ariaHidden":397},[396],[367,6069,6071,6074,6114,6117,6120],{"className":6070},[401],[367,6072],{"className":6073,"style":651},[405],[367,6075,6077,6080],{"className":6076},[410],[367,6078,5770],{"className":6079,"style":1021},[410,411],[367,6081,6083],{"className":6082},[661],[367,6084,6086,6106],{"className":6085},[665,666],[367,6087,6089,6103],{"className":6088},[670],[367,6090,6092],{"className":6091,"style":675},[674],[367,6093,6094,6097],{"style":5841},[367,6095],{"className":6096,"style":683},[682],[367,6098,6100],{"className":6099},[687,688,689,690],[367,6101,619],{"className":6102},[410,411,690],[367,6104,698],{"className":6105},[697],[367,6107,6109],{"className":6108},[670],[367,6110,6112],{"className":6111,"style":705},[674],[367,6113],{},[367,6115],{"className":6116,"style":712},[711],[367,6118,623],{"className":6119},[716],[367,6121],{"className":6122,"style":712},[711],[367,6124,6126,6129,6132,6135,6175],{"className":6125},[401],[367,6127],{"className":6128,"style":726},[405],[367,6130,5734],{"className":6131,"style":5749},[410,411],[367,6133,629],{"className":6134},[733],[367,6136,6138,6141],{"className":6137},[410],[367,6139,388],{"className":6140},[410,411],[367,6142,6144],{"className":6143},[661],[367,6145,6147,6167],{"className":6146},[665,666],[367,6148,6150,6164],{"className":6149},[670],[367,6151,6153],{"className":6152,"style":675},[674],[367,6154,6155,6158],{"style":678},[367,6156],{"className":6157,"style":683},[682],[367,6159,6161],{"className":6160},[687,688,689,690],[367,6162,619],{"className":6163},[410,411,690],[367,6165,698],{"className":6166},[697],[367,6168,6170],{"className":6169},[670],[367,6171,6173],{"className":6172,"style":705},[674],[367,6174],{},[367,6176,638],{"className":6177},[777],"，或依赖状态-动作对 ",[367,6180,6182,6222],{"className":6181},[370],[367,6183,6185],{"className":6184},[374],[376,6186,6187],{"xmlns":378},[380,6188,6189,6219],{},[383,6190,6191,6197,6199,6201,6203,6209,6211,6217],{},[613,6192,6193,6195],{},[386,6194,5770],{},[386,6196,619],{},[621,6198,623],{},[386,6200,5734],{},[621,6202,629],{"stretchy":628},[613,6204,6205,6207],{},[386,6206,388],{},[386,6208,619],{},[621,6210,2656],{"separator":397},[613,6212,6213,6215],{},[386,6214,216],{},[386,6216,619],{},[621,6218,638],{"stretchy":628},[390,6220,6221],{"encoding":392},"r_t = R(s_t, a_t)",[367,6223,6225,6280],{"className":6224,"ariaHidden":397},[396],[367,6226,6228,6231,6271,6274,6277],{"className":6227},[401],[367,6229],{"className":6230,"style":651},[405],[367,6232,6234,6237],{"className":6233},[410],[367,6235,5770],{"className":6236,"style":1021},[410,411],[367,6238,6240],{"className":6239},[661],[367,6241,6243,6263],{"className":6242},[665,666],[367,6244,6246,6260],{"className":6245},[670],[367,6247,6249],{"className":6248,"style":675},[674],[367,6250,6251,6254],{"style":5841},[367,6252],{"className":6253,"style":683},[682],[367,6255,6257],{"className":6256},[687,688,689,690],[367,6258,619],{"className":6259},[410,411,690],[367,6261,698],{"className":6262},[697],[367,6264,6266],{"className":6265},[670],[367,6267,6269],{"className":6268,"style":705},[674],[367,6270],{},[367,6272],{"className":6273,"style":712},[711],[367,6275,623],{"className":6276},[716],[367,6278],{"className":6279,"style":712},[711],[367,6281,6283,6286,6289,6292,6332,6335,6338,6378],{"className":6282},[401],[367,6284],{"className":6285,"style":726},[405],[367,6287,5734],{"className":6288,"style":5749},[410,411],[367,6290,629],{"className":6291},[733],[367,6293,6295,6298],{"className":6294},[410],[367,6296,388],{"className":6297},[410,411],[367,6299,6301],{"className":6300},[661],[367,6302,6304,6324],{"className":6303},[665,666],[367,6305,6307,6321],{"className":6306},[670],[367,6308,6310],{"className":6309,"style":675},[674],[367,6311,6312,6315],{"style":678},[367,6313],{"className":6314,"style":683},[682],[367,6316,6318],{"className":6317},[687,688,689,690],[367,6319,619],{"className":6320},[410,411,690],[367,6322,698],{"className":6323},[697],[367,6325,6327],{"className":6326},[670],[367,6328,6330],{"className":6329,"style":705},[674],[367,6331],{},[367,6333,2656],{"className":6334},[2685],[367,6336],{"className":6337,"style":1773},[711],[367,6339,6341,6344],{"className":6340},[410],[367,6342,216],{"className":6343},[410,411],[367,6345,6347],{"className":6346},[661],[367,6348,6350,6370],{"className":6349},[665,666],[367,6351,6353,6367],{"className":6352},[670],[367,6354,6356],{"className":6355,"style":675},[674],[367,6357,6358,6361],{"style":678},[367,6359],{"className":6360,"style":683},[682],[367,6362,6364],{"className":6363},[687,688,689,690],[367,6365,619],{"className":6366},[410,411,690],[367,6368,698],{"className":6369},[697],[367,6371,6373],{"className":6372},[670],[367,6374,6376],{"className":6375,"style":705},[674],[367,6377],{},[367,6379,638],{"className":6380},[777],[212,6382,6383,6384,6428],{},"智能体的目标是最大化某种「沿轨迹累积的奖励」，但这话实际可以有几种含义。我们对所有这些情形统一记作 ",[367,6385,6387,6407],{"className":6386},[370],[367,6388,6390],{"className":6389},[374],[376,6391,6392],{"xmlns":378},[380,6393,6394,6404],{},[383,6395,6396,6398,6400,6402],{},[386,6397,5734],{},[621,6399,629],{"stretchy":628},[386,6401,4325],{},[621,6403,638],{"stretchy":628},[390,6405,6406],{"encoding":392},"R(\\tau)",[367,6408,6410],{"className":6409,"ariaHidden":397},[396],[367,6411,6413,6416,6419,6422,6425],{"className":6412},[401],[367,6414],{"className":6415,"style":726},[405],[367,6417,5734],{"className":6418,"style":5749},[410,411],[367,6420,629],{"className":6421},[733],[367,6423,4325],{"className":6424,"style":4341},[410,411],[367,6426,638],{"className":6427},[777],"：具体指哪一种，要么能从上下文分辨，要么无关紧要（因为同样的方程适用于所有情形）。",[212,6430,6431,6432,6435],{},"一种回报是",[315,6433,6434],{},"有限视界不折扣回报（finite-horizon undiscounted return）","：固定步数窗口内获得的奖励之和：",[367,6437,6439],{"className":6438},[598],[367,6440,6442,6485],{"className":6441},[370],[367,6443,6445],{"className":6444},[374],[376,6446,6447],{"xmlns":378,"display":607},[380,6448,6449,6482],{},[383,6450,6451,6453,6455,6457,6459,6461,6476],{},[386,6452,5734],{},[621,6454,629],{"stretchy":628},[386,6456,4325],{},[621,6458,638],{"stretchy":628},[621,6460,623],{},[3612,6462,6463,6465,6473],{},[621,6464,3616],{},[383,6466,6467,6469,6471],{},[386,6468,619],{},[621,6470,623],{},[2932,6472,1083],{},[386,6474,6475],{},"T",[613,6477,6478,6480],{},[386,6479,5770],{},[386,6481,619],{},[390,6483,6484],{"encoding":392},"R(\\tau) = \\sum_{t=0}^T r_t",[367,6486,6488,6515],{"className":6487,"ariaHidden":397},[396],[367,6489,6491,6494,6497,6500,6503,6506,6509,6512],{"className":6490},[401],[367,6492],{"className":6493,"style":726},[405],[367,6495,5734],{"className":6496,"style":5749},[410,411],[367,6498,629],{"className":6499},[733],[367,6501,4325],{"className":6502,"style":4341},[410,411],[367,6504,638],{"className":6505},[777],[367,6507],{"className":6508,"style":712},[711],[367,6510,623],{"className":6511},[716],[367,6513],{"className":6514,"style":712},[711],[367,6516,6518,6522,6589,6592],{"className":6517},[401],[367,6519],{"className":6520,"style":6521},[405],"height:3.0954em;vertical-align:-1.2671em;",[367,6523,6525],{"className":6524},[1653,3882],[367,6526,6528,6580],{"className":6527},[665,666],[367,6529,6531,6577],{"className":6530},[670],[367,6532,6535,6556,6566],{"className":6533,"style":6534},[674],"height:1.8283em;",[367,6536,6538,6541],{"style":6537},"top:-1.8829em;margin-left:0em;",[367,6539],{"className":6540,"style":3899},[682],[367,6542,6544],{"className":6543},[687,688,689,690],[367,6545,6547,6550,6553],{"className":6546},[410,690],[367,6548,619],{"className":6549},[410,411,690],[367,6551,623],{"className":6552},[716,690],[367,6554,1083],{"className":6555},[410,690],[367,6557,6558,6561],{"style":3917},[367,6559],{"className":6560,"style":3899},[682],[367,6562,6563],{},[367,6564,3616],{"className":6565},[1653,3926,3927],[367,6567,6568,6571],{"style":3930},[367,6569],{"className":6570,"style":3899},[682],[367,6572,6574],{"className":6573},[687,688,689,690],[367,6575,6475],{"className":6576,"style":1922},[410,411,690],[367,6578,698],{"className":6579},[697],[367,6581,6583],{"className":6582},[670],[367,6584,6587],{"className":6585,"style":6586},[674],"height:1.2671em;",[367,6588],{},[367,6590],{"className":6591,"style":1773},[711],[367,6593,6595,6598],{"className":6594},[410],[367,6596,5770],{"className":6597,"style":1021},[410,411],[367,6599,6601],{"className":6600},[661],[367,6602,6604,6624],{"className":6603},[665,666],[367,6605,6607,6621],{"className":6606},[670],[367,6608,6610],{"className":6609,"style":675},[674],[367,6611,6612,6615],{"style":5841},[367,6613],{"className":6614,"style":683},[682],[367,6616,6618],{"className":6617},[687,688,689,690],[367,6619,619],{"className":6620},[410,411,690],[367,6622,698],{"className":6623},[697],[367,6625,6627],{"className":6626},[670],[367,6628,6630],{"className":6629,"style":705},[674],[367,6631],{},[212,6633,6634,6635,6638,6639,6642,6643,594],{},"另一种是",[315,6636,6637],{},"无限视界折扣回报（infinite-horizon discounted return）","：智能体",[243,6640,6641],{},"曾经","获得的全部奖励之和，但按奖励到来时刻的远近打折。这种回报形式引入了折扣因子 ",[367,6644,6646,6674],{"className":6645},[370],[367,6647,6649],{"className":6648},[374],[376,6650,6651],{"xmlns":378},[380,6652,6653,6671],{},[383,6654,6655,6658,6661,6663,6665,6667,6669],{},[386,6656,6657],{},"γ",[621,6659,6660],{},"∈",[621,6662,629],{"stretchy":628},[2932,6664,1083],{},[621,6666,2656],{"separator":397},[2932,6668,3603],{},[621,6670,638],{"stretchy":628},[390,6672,6673],{"encoding":392},"\\gamma \\in (0,1)",[367,6675,6677,6697],{"className":6676,"ariaHidden":397},[396],[367,6678,6680,6684,6688,6691,6694],{"className":6679},[401],[367,6681],{"className":6682,"style":6683},[405],"height:0.7335em;vertical-align:-0.1944em;",[367,6685,6657],{"className":6686,"style":6687},[410,411],"margin-right:0.0556em;",[367,6689],{"className":6690,"style":712},[711],[367,6692,6660],{"className":6693},[716],[367,6695],{"className":6696,"style":712},[711],[367,6698,6700,6703,6706,6709,6712,6715,6718],{"className":6699},[401],[367,6701],{"className":6702,"style":726},[405],[367,6704,629],{"className":6705},[733],[367,6707,1083],{"className":6708},[410],[367,6710,2656],{"className":6711},[2685],[367,6713],{"className":6714,"style":1773},[711],[367,6716,3603],{"className":6717},[410],[367,6719,638],{"className":6720},[777],[367,6722,6724],{"className":6723},[598],[367,6725,6727,6775],{"className":6726},[370],[367,6728,6730],{"className":6729},[374],[376,6731,6732],{"xmlns":378,"display":607},[380,6733,6734,6772],{},[383,6735,6736,6738,6740,6742,6744,6746,6760,6766],{},[386,6737,5734],{},[621,6739,629],{"stretchy":628},[386,6741,4325],{},[621,6743,638],{"stretchy":628},[621,6745,623],{},[3612,6747,6748,6750,6758],{},[621,6749,3616],{},[383,6751,6752,6754,6756],{},[386,6753,619],{},[621,6755,623],{},[2932,6757,1083],{},[386,6759,2653],{"mathvariant":845},[3653,6761,6762,6764],{},[386,6763,6657],{},[386,6765,619],{},[613,6767,6768,6770],{},[386,6769,5770],{},[386,6771,619],{},[390,6773,6774],{"encoding":392},"R(\\tau) = \\sum_{t=0}^{\\infty} \\gamma^t r_t",[367,6776,6778,6805],{"className":6777,"ariaHidden":397},[396],[367,6779,6781,6784,6787,6790,6793,6796,6799,6802],{"className":6780},[401],[367,6782],{"className":6783,"style":726},[405],[367,6785,5734],{"className":6786,"style":5749},[410,411],[367,6788,629],{"className":6789},[733],[367,6791,4325],{"className":6792,"style":4341},[410,411],[367,6794,638],{"className":6795},[777],[367,6797],{"className":6798,"style":712},[711],[367,6800,623],{"className":6801},[716],[367,6803],{"className":6804,"style":712},[711],[367,6806,6808,6812,6880,6883,6914],{"className":6807},[401],[367,6809],{"className":6810,"style":6811},[405],"height:2.9185em;vertical-align:-1.2671em;",[367,6813,6815],{"className":6814},[1653,3882],[367,6816,6818,6872],{"className":6817},[665,666],[367,6819,6821,6869],{"className":6820},[670],[367,6822,6825,6845,6855],{"className":6823,"style":6824},[674],"height:1.6514em;",[367,6826,6827,6830],{"style":6537},[367,6828],{"className":6829,"style":3899},[682],[367,6831,6833],{"className":6832},[687,688,689,690],[367,6834,6836,6839,6842],{"className":6835},[410,690],[367,6837,619],{"className":6838},[410,411,690],[367,6840,623],{"className":6841},[716,690],[367,6843,1083],{"className":6844},[410,690],[367,6846,6847,6850],{"style":3917},[367,6848],{"className":6849,"style":3899},[682],[367,6851,6852],{},[367,6853,3616],{"className":6854},[1653,3926,3927],[367,6856,6857,6860],{"style":3930},[367,6858],{"className":6859,"style":3899},[682],[367,6861,6863],{"className":6862},[687,688,689,690],[367,6864,6866],{"className":6865},[410,690],[367,6867,2653],{"className":6868},[410,690],[367,6870,698],{"className":6871},[697],[367,6873,6875],{"className":6874},[670],[367,6876,6878],{"className":6877,"style":6586},[674],[367,6879],{},[367,6881],{"className":6882,"style":1773},[711],[367,6884,6886,6889],{"className":6885},[410],[367,6887,6657],{"className":6888,"style":6687},[410,411],[367,6890,6892],{"className":6891},[661],[367,6893,6895],{"className":6894},[665],[367,6896,6898],{"className":6897},[670],[367,6899,6902],{"className":6900,"style":6901},[674],"height:0.8436em;",[367,6903,6905,6908],{"style":6904},"top:-3.113em;margin-right:0.05em;",[367,6906],{"className":6907,"style":683},[682],[367,6909,6911],{"className":6910},[687,688,689,690],[367,6912,619],{"className":6913},[410,411,690],[367,6915,6917,6920],{"className":6916},[410],[367,6918,5770],{"className":6919,"style":1021},[410,411],[367,6921,6923],{"className":6922},[661],[367,6924,6926,6946],{"className":6925},[665,666],[367,6927,6929,6943],{"className":6928},[670],[367,6930,6932],{"className":6931,"style":675},[674],[367,6933,6934,6937],{"style":5841},[367,6935],{"className":6936,"style":683},[682],[367,6938,6940],{"className":6939},[687,688,689,690],[367,6941,619],{"className":6942},[410,411,690],[367,6944,698],{"className":6945},[697],[367,6947,6949],{"className":6948},[670],[367,6950,6952],{"className":6951,"style":705},[674],[367,6953],{},[212,6955,6956,6957,6960,6961,6966],{},"可我们为什么会想要折扣因子？难道不就是把",[243,6958,6959],{},"所有","奖励都拿到手吗？确实想，但折扣因子既直观又便于数学处理。直观层面：现在的现金好过以后的现金。数学层面：无限视界的奖励和",[216,6962,6965],{"href":6963,"rel":6964},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FConvergent_series",[220],"可能不收敛","到有限值，在方程里难以处理；而有了折扣因子，再加上合理条件，无穷和就收敛了。",[208,6968,6969],{"title":462},[212,6970,6971,6972,6975],{},"这两种回报形式在 RL 形式化里界限分明，但深度 RL 的实践常把界线弄得相当模糊：比如我们经常把算法设定为优化不折扣的回报，却在估计",[315,6973,6974],{},"价值函数","时使用折扣因子。",[357,6977,6979],{"id":6978},"rl-问题","RL 问题",[212,6981,6982,6983,6986],{},"无论选哪种回报度量（无限视界折扣，还是有限视界不折扣），也无论策略怎么选，RL 的目标都是：挑出一个策略，使智能体按它行动时的",[315,6984,6985],{},"期望回报（expected return）"," 最大。",[212,6988,6989],{},"要谈期望回报，得先谈轨迹上的概率分布。",[212,6991,6992,6993,7021],{},"假设环境转移和策略都是随机型的。这时，一条 ",[367,6994,6996,7009],{"className":6995},[370],[367,6997,6999],{"className":6998},[374],[376,7000,7001],{"xmlns":378},[380,7002,7003,7007],{},[383,7004,7005],{},[386,7006,6475],{},[390,7008,6475],{"encoding":392},[367,7010,7012],{"className":7011,"ariaHidden":397},[396],[367,7013,7015,7018],{"className":7014},[401],[367,7016],{"className":7017,"style":2333},[405],[367,7019,6475],{"className":7020,"style":1922},[410,411]," 步轨迹的概率是：",[367,7023,7025],{"className":7024},[598],[367,7026,7028,7145],{"className":7027},[370],[367,7029,7031],{"className":7030},[374],[376,7032,7033],{"xmlns":378,"display":607},[380,7034,7035,7142],{},[383,7036,7037,7039,7041,7043,7045,7047,7049,7051,7057,7059,7065,7067,7088,7090,7092,7104,7106,7112,7114,7120,7122,7124,7126,7132,7134,7140],{},[386,7038,1895],{},[621,7040,629],{"stretchy":628},[386,7042,4325],{},[386,7044,846],{"mathvariant":845},[386,7046,795],{},[621,7048,638],{"stretchy":628},[621,7050,623],{},[613,7052,7053,7055],{},[386,7054,4724],{},[2932,7056,1083],{},[621,7058,629],{"stretchy":628},[613,7060,7061,7063],{},[386,7062,388],{},[2932,7064,1083],{},[621,7066,638],{"stretchy":628},[3612,7068,7069,7072,7080],{},[621,7070,7071],{},"∏",[383,7073,7074,7076,7078],{},[386,7075,619],{},[621,7077,623],{},[2932,7079,1083],{},[383,7081,7082,7084,7086],{},[386,7083,6475],{},[621,7085,2650],{},[2932,7087,3603],{},[386,7089,1895],{},[621,7091,629],{"stretchy":628},[613,7093,7094,7096],{},[386,7095,388],{},[383,7097,7098,7100,7102],{},[386,7099,619],{},[621,7101,3029],{},[2932,7103,3603],{},[386,7105,846],{"mathvariant":845},[613,7107,7108,7110],{},[386,7109,388],{},[386,7111,619],{},[621,7113,2656],{"separator":397},[613,7115,7116,7118],{},[386,7117,216],{},[386,7119,619],{},[621,7121,638],{"stretchy":628},[386,7123,795],{},[621,7125,629],{"stretchy":628},[613,7127,7128,7130],{},[386,7129,216],{},[386,7131,619],{},[386,7133,846],{"mathvariant":845},[613,7135,7136,7138],{},[386,7137,388],{},[386,7139,619],{},[621,7141,638],{"stretchy":628},[390,7143,7144],{"encoding":392},"P(\\tau|\\pi) = \\rho_0 (s_0) \\prod_{t=0}^{T-1} P(s_{t+1} | s_t, a_t) \\pi(a_t | s_t)",[367,7146,7148,7181],{"className":7147,"ariaHidden":397},[396],[367,7149,7151,7154,7157,7160,7163,7166,7169,7172,7175,7178],{"className":7150},[401],[367,7152],{"className":7153,"style":726},[405],[367,7155,1895],{"className":7156,"style":1922},[410,411],[367,7158,629],{"className":7159},[733],[367,7161,4325],{"className":7162,"style":4341},[410,411],[367,7164,846],{"className":7165},[410],[367,7167,795],{"className":7168,"style":811},[410,411],[367,7170,638],{"className":7171},[777],[367,7173],{"className":7174,"style":712},[711],[367,7176,623],{"className":7177},[716],[367,7179],{"className":7180,"style":712},[711],[367,7182,7184,7187,7227,7230,7270,7273,7276,7349,7352,7355,7358,7407,7410,7450,7453,7456,7496,7499,7502,7505,7545,7548,7588],{"className":7183},[401],[367,7185],{"className":7186,"style":6521},[405],[367,7188,7190,7193],{"className":7189},[410],[367,7191,4724],{"className":7192},[410,411],[367,7194,7196],{"className":7195},[661],[367,7197,7199,7219],{"className":7198},[665,666],[367,7200,7202,7216],{"className":7201},[670],[367,7203,7205],{"className":7204,"style":4456},[674],[367,7206,7207,7210],{"style":678},[367,7208],{"className":7209,"style":683},[682],[367,7211,7213],{"className":7212},[687,688,689,690],[367,7214,1083],{"className":7215},[410,690],[367,7217,698],{"className":7218},[697],[367,7220,7222],{"className":7221},[670],[367,7223,7225],{"className":7224,"style":705},[674],[367,7226],{},[367,7228,629],{"className":7229},[733],[367,7231,7233,7236],{"className":7232},[410],[367,7234,388],{"className":7235},[410,411],[367,7237,7239],{"className":7238},[661],[367,7240,7242,7262],{"className":7241},[665,666],[367,7243,7245,7259],{"className":7244},[670],[367,7246,7248],{"className":7247,"style":4456},[674],[367,7249,7250,7253],{"style":678},[367,7251],{"className":7252,"style":683},[682],[367,7254,7256],{"className":7255},[687,688,689,690],[367,7257,1083],{"className":7258},[410,690],[367,7260,698],{"className":7261},[697],[367,7263,7265],{"className":7264},[670],[367,7266,7268],{"className":7267,"style":705},[674],[367,7269],{},[367,7271,638],{"className":7272},[777],[367,7274],{"className":7275,"style":1773},[711],[367,7277,7279],{"className":7278},[1653,3882],[367,7280,7282,7341],{"className":7281},[665,666],[367,7283,7285,7338],{"className":7284},[670],[367,7286,7288,7308,7318],{"className":7287,"style":6534},[674],[367,7289,7290,7293],{"style":6537},[367,7291],{"className":7292,"style":3899},[682],[367,7294,7296],{"className":7295},[687,688,689,690],[367,7297,7299,7302,7305],{"className":7298},[410,690],[367,7300,619],{"className":7301},[410,411,690],[367,7303,623],{"className":7304},[716,690],[367,7306,1083],{"className":7307},[410,690],[367,7309,7310,7313],{"style":3917},[367,7311],{"className":7312,"style":3899},[682],[367,7314,7315],{},[367,7316,7071],{"className":7317},[1653,3926,3927],[367,7319,7320,7323],{"style":3930},[367,7321],{"className":7322,"style":3899},[682],[367,7324,7326],{"className":7325},[687,688,689,690],[367,7327,7329,7332,7335],{"className":7328},[410,690],[367,7330,6475],{"className":7331,"style":1922},[410,411,690],[367,7333,2650],{"className":7334},[3136,690],[367,7336,3603],{"className":7337},[410,690],[367,7339,698],{"className":7340},[697],[367,7342,7344],{"className":7343},[670],[367,7345,7347],{"className":7346,"style":6586},[674],[367,7348],{},[367,7350],{"className":7351,"style":1773},[711],[367,7353,1895],{"className":7354,"style":1922},[410,411],[367,7356,629],{"className":7357},[733],[367,7359,7361,7364],{"className":7360},[410],[367,7362,388],{"className":7363},[410,411],[367,7365,7367],{"className":7366},[661],[367,7368,7370,7399],{"className":7369},[665,666],[367,7371,7373,7396],{"className":7372},[670],[367,7374,7376],{"className":7375,"style":4456},[674],[367,7377,7378,7381],{"style":678},[367,7379],{"className":7380,"style":683},[682],[367,7382,7384],{"className":7383},[687,688,689,690],[367,7385,7387,7390,7393],{"className":7386},[410,690],[367,7388,619],{"className":7389},[410,411,690],[367,7391,3029],{"className":7392},[3136,690],[367,7394,3603],{"className":7395},[410,690],[367,7397,698],{"className":7398},[697],[367,7400,7402],{"className":7401},[670],[367,7403,7405],{"className":7404,"style":5172},[674],[367,7406],{},[367,7408,846],{"className":7409},[410],[367,7411,7413,7416],{"className":7412},[410],[367,7414,388],{"className":7415},[410,411],[367,7417,7419],{"className":7418},[661],[367,7420,7422,7442],{"className":7421},[665,666],[367,7423,7425,7439],{"className":7424},[670],[367,7426,7428],{"className":7427,"style":675},[674],[367,7429,7430,7433],{"style":678},[367,7431],{"className":7432,"style":683},[682],[367,7434,7436],{"className":7435},[687,688,689,690],[367,7437,619],{"className":7438},[410,411,690],[367,7440,698],{"className":7441},[697],[367,7443,7445],{"className":7444},[670],[367,7446,7448],{"className":7447,"style":705},[674],[367,7449],{},[367,7451,2656],{"className":7452},[2685],[367,7454],{"className":7455,"style":1773},[711],[367,7457,7459,7462],{"className":7458},[410],[367,7460,216],{"className":7461},[410,411],[367,7463,7465],{"className":7464},[661],[367,7466,7468,7488],{"className":7467},[665,666],[367,7469,7471,7485],{"className":7470},[670],[367,7472,7474],{"className":7473,"style":675},[674],[367,7475,7476,7479],{"style":678},[367,7477],{"className":7478,"style":683},[682],[367,7480,7482],{"className":7481},[687,688,689,690],[367,7483,619],{"className":7484},[410,411,690],[367,7486,698],{"className":7487},[697],[367,7489,7491],{"className":7490},[670],[367,7492,7494],{"className":7493,"style":705},[674],[367,7495],{},[367,7497,638],{"className":7498},[777],[367,7500,795],{"className":7501,"style":811},[410,411],[367,7503,629],{"className":7504},[733],[367,7506,7508,7511],{"className":7507},[410],[367,7509,216],{"className":7510},[410,411],[367,7512,7514],{"className":7513},[661],[367,7515,7517,7537],{"className":7516},[665,666],[367,7518,7520,7534],{"className":7519},[670],[367,7521,7523],{"className":7522,"style":675},[674],[367,7524,7525,7528],{"style":678},[367,7526],{"className":7527,"style":683},[682],[367,7529,7531],{"className":7530},[687,688,689,690],[367,7532,619],{"className":7533},[410,411,690],[367,7535,698],{"className":7536},[697],[367,7538,7540],{"className":7539},[670],[367,7541,7543],{"className":7542,"style":705},[674],[367,7544],{},[367,7546,846],{"className":7547},[410],[367,7549,7551,7554],{"className":7550},[410],[367,7552,388],{"className":7553},[410,411],[367,7555,7557],{"className":7556},[661],[367,7558,7560,7580],{"className":7559},[665,666],[367,7561,7563,7577],{"className":7562},[670],[367,7564,7566],{"className":7565,"style":675},[674],[367,7567,7568,7571],{"style":678},[367,7569],{"className":7570,"style":683},[682],[367,7572,7574],{"className":7573},[687,688,689,690],[367,7575,619],{"className":7576},[410,411,690],[367,7578,698],{"className":7579},[697],[367,7581,7583],{"className":7582},[670],[367,7584,7586],{"className":7585,"style":705},[674],[367,7587],{},[367,7589,638],{"className":7590},[777],[212,7592,7593,7594,594],{},"期望回报（无论按哪种度量）记作 ",[367,7595,7597,7618],{"className":7596},[370],[367,7598,7600],{"className":7599},[374],[376,7601,7602],{"xmlns":378},[380,7603,7604,7615],{},[383,7605,7606,7609,7611,7613],{},[386,7607,7608],{},"J",[621,7610,629],{"stretchy":628},[386,7612,795],{},[621,7614,638],{"stretchy":628},[390,7616,7617],{"encoding":392},"J(\\pi)",[367,7619,7621],{"className":7620,"ariaHidden":397},[396],[367,7622,7624,7627,7631,7634,7637],{"className":7623},[401],[367,7625],{"className":7626,"style":726},[405],[367,7628,7608],{"className":7629,"style":7630},[410,411],"margin-right:0.0962em;",[367,7632,629],{"className":7633},[733],[367,7635,795],{"className":7636,"style":811},[410,411],[367,7638,638],{"className":7639},[777],[367,7641,7643],{"className":7642},[598],[367,7644,7646,7728],{"className":7645},[370],[367,7647,7649],{"className":7648},[374],[376,7650,7651],{"xmlns":378,"display":607},[380,7652,7653,7725],{},[383,7654,7655,7657,7659,7661,7663,7665,7672,7674,7676,7678,7680,7682,7684,7686,7688,7690,7692,7694,7713,7715,7717,7719,7721,7723],{},[386,7656,7608],{},[621,7658,629],{"stretchy":628},[386,7660,795],{},[621,7662,638],{"stretchy":628},[621,7664,623],{},[613,7666,7667,7670],{},[621,7668,7669],{},"∫",[386,7671,4325],{},[386,7673,1895],{},[621,7675,629],{"stretchy":628},[386,7677,4325],{},[386,7679,846],{"mathvariant":845},[386,7681,795],{},[621,7683,638],{"stretchy":628},[386,7685,5734],{},[621,7687,629],{"stretchy":628},[386,7689,4325],{},[621,7691,638],{"stretchy":628},[621,7693,623],{},[386,7695,7696],{},[7697,7698,7699,7705],"munder",{},[621,7700,7701],{},[386,7702,7704],{"mathvariant":7703},"double-struck","E",[383,7706,7707,7709,7711],{},[386,7708,4325],{},[621,7710,835],{},[386,7712,795],{},[621,7714,2047],{"stretchy":628},[386,7716,5734],{},[621,7718,629],{"stretchy":628},[386,7720,4325],{},[621,7722,638],{"stretchy":628},[621,7724,2062],{"stretchy":628},[390,7726,7727],{"encoding":392},"J(\\pi) = \\int_{\\tau} P(\\tau|\\pi) R(\\tau) = \\underset{\\tau\\sim \\pi}{\\mathbb{E}}[R(\\tau)]",[367,7729,7731,7758,7854],{"className":7730,"ariaHidden":397},[396],[367,7732,7734,7737,7740,7743,7746,7749,7752,7755],{"className":7733},[401],[367,7735],{"className":7736,"style":726},[405],[367,7738,7608],{"className":7739,"style":7630},[410,411],[367,7741,629],{"className":7742},[733],[367,7744,795],{"className":7745,"style":811},[410,411],[367,7747,638],{"className":7748},[777],[367,7750],{"className":7751,"style":712},[711],[367,7753,623],{"className":7754},[716],[367,7756],{"className":7757,"style":712},[711],[367,7759,7761,7765,7812,7815,7818,7821,7824,7827,7830,7833,7836,7839,7842,7845,7848,7851],{"className":7760},[401],[367,7762],{"className":7763,"style":7764},[405],"height:2.2719em;vertical-align:-0.9119em;",[367,7766,7768,7772],{"className":7767},[1653],[367,7769,7669],{"className":7770,"style":7771},[1653,3926,3927],"margin-right:0.4445em;position:relative;top:-0.0011em;",[367,7773,7775],{"className":7774},[661],[367,7776,7778,7803],{"className":7777},[665,666],[367,7779,7781,7800],{"className":7780},[670],[367,7782,7785],{"className":7783,"style":7784},[674],"height:-0.6106em;",[367,7786,7788,7791],{"style":7787},"top:-1.7881em;margin-left:-0.4445em;margin-right:0.05em;",[367,7789],{"className":7790,"style":683},[682],[367,7792,7794],{"className":7793},[687,688,689,690],[367,7795,7797],{"className":7796},[410,690],[367,7798,4325],{"className":7799,"style":4341},[410,411,690],[367,7801,698],{"className":7802},[697],[367,7804,7806],{"className":7805},[670],[367,7807,7810],{"className":7808,"style":7809},[674],"height:0.9119em;",[367,7811],{},[367,7813],{"className":7814,"style":1773},[711],[367,7816,1895],{"className":7817,"style":1922},[410,411],[367,7819,629],{"className":7820},[733],[367,7822,4325],{"className":7823,"style":4341},[410,411],[367,7825,846],{"className":7826},[410],[367,7828,795],{"className":7829,"style":811},[410,411],[367,7831,638],{"className":7832},[777],[367,7834,5734],{"className":7835,"style":5749},[410,411],[367,7837,629],{"className":7838},[733],[367,7840,4325],{"className":7841,"style":4341},[410,411],[367,7843,638],{"className":7844},[777],[367,7846],{"className":7847,"style":712},[711],[367,7849,623],{"className":7850},[716],[367,7852],{"className":7853,"style":712},[711],[367,7855,7857,7861,7922,7925,7928,7931,7934],{"className":7856},[401],[367,7858],{"className":7859,"style":7860},[405],"height:1.45em;vertical-align:-0.7em;",[367,7862,7864],{"className":7863},[410],[367,7865,7867],{"className":7866},[1653,3882],[367,7868,7870,7913],{"className":7869},[665,666],[367,7871,7873,7910],{"className":7872},[670],[367,7874,7877,7898],{"className":7875,"style":7876},[674],"height:0.6889em;",[367,7878,7880,7883],{"style":7879},"top:-2.4em;margin-left:0em;",[367,7881],{"className":7882,"style":1197},[682],[367,7884,7886],{"className":7885},[687,688,689,690],[367,7887,7889,7892,7895],{"className":7888},[410,690],[367,7890,4325],{"className":7891,"style":4341},[410,411,690],[367,7893,835],{"className":7894},[716,690],[367,7896,795],{"className":7897,"style":811},[410,411,690],[367,7899,7901,7904],{"style":7900},"top:-3em;",[367,7902],{"className":7903,"style":1197},[682],[367,7905,7906],{},[367,7907,7704],{"className":7908},[1653,7909],"mathbb",[367,7911,698],{"className":7912},[697],[367,7914,7916],{"className":7915},[670],[367,7917,7920],{"className":7918,"style":7919},[674],"height:0.7em;",[367,7921],{},[367,7923,2047],{"className":7924},[733],[367,7926,5734],{"className":7927,"style":5749},[410,411],[367,7929,629],{"className":7930},[733],[367,7932,4325],{"className":7933,"style":4341},[410,411],[367,7935,7937],{"className":7936},[777],")]",[212,7939,7940],{},"RL 的中心优化问题于是可以写成：",[367,7942,7944],{"className":7943},[598],[367,7945,7947,7992],{"className":7946},[370],[367,7948,7950],{"className":7949},[374],[376,7951,7952],{"xmlns":378,"display":607},[380,7953,7954,7989],{},[383,7955,7956,7963,7965,7968,7970,7981,7983,7985,7987],{},[3653,7957,7958,7960],{},[386,7959,795],{},[621,7961,7962],{},"∗",[621,7964,623],{},[386,7966,7967],{},"arg",[621,7969,1637],{},[7697,7971,7972,7979],{},[383,7973,7974,7977],{},[386,7975,7976],{},"max",[621,7978,1637],{},[386,7980,795],{},[386,7982,7608],{},[621,7984,629],{"stretchy":628},[386,7986,795],{},[621,7988,638],{"stretchy":628},[390,7990,7991],{"encoding":392},"\\pi^* = \\arg \\max_{\\pi} J(\\pi)",[367,7993,7995,8040],{"className":7994,"ariaHidden":397},[396],[367,7996,7998,8002,8031,8034,8037],{"className":7997},[401],[367,7999],{"className":8000,"style":8001},[405],"height:0.7387em;",[367,8003,8005,8008],{"className":8004},[410],[367,8006,795],{"className":8007,"style":811},[410,411],[367,8009,8011],{"className":8010},[661],[367,8012,8014],{"className":8013},[665],[367,8015,8017],{"className":8016},[670],[367,8018,8020],{"className":8019,"style":8001},[674],[367,8021,8022,8025],{"style":6904},[367,8023],{"className":8024,"style":683},[682],[367,8026,8028],{"className":8027},[687,688,689,690],[367,8029,7962],{"className":8030},[3136,690],[367,8032],{"className":8033,"style":712},[711],[367,8035,623],{"className":8036},[716],[367,8038],{"className":8039,"style":712},[711],[367,8041,8043,8046,8052,8055,8102,8105,8108,8111,8114],{"className":8042},[401],[367,8044],{"className":8045,"style":7860},[405],[367,8047,8049,8050],{"className":8048},[1653],"ar",[367,8051,1769],{"style":1768},[367,8053],{"className":8054,"style":1773},[711],[367,8056,8058],{"className":8057},[1653,3882],[367,8059,8061,8094],{"className":8060},[665,666],[367,8062,8064,8091],{"className":8063},[670],[367,8065,8067,8081],{"className":8066,"style":406},[674],[367,8068,8069,8072],{"style":7879},[367,8070],{"className":8071,"style":1197},[682],[367,8073,8075],{"className":8074},[687,688,689,690],[367,8076,8078],{"className":8077},[410,690],[367,8079,795],{"className":8080,"style":811},[410,411,690],[367,8082,8083,8086],{"style":7900},[367,8084],{"className":8085,"style":1197},[682],[367,8087,8088],{},[367,8089,7976],{"className":8090},[1653],[367,8092,698],{"className":8093},[697],[367,8095,8097],{"className":8096},[670],[367,8098,8100],{"className":8099,"style":7919},[674],[367,8101],{},[367,8103],{"className":8104,"style":1773},[711],[367,8106,7608],{"className":8107,"style":7630},[410,411],[367,8109,629],{"className":8110},[733],[367,8112,795],{"className":8113,"style":811},[410,411],[367,8115,638],{"className":8116},[777],[212,8118,8119,8120,8180,8181,272],{},"其中的 ",[367,8121,8123,8141],{"className":8122},[370],[367,8124,8126],{"className":8125},[374],[376,8127,8128],{"xmlns":378},[380,8129,8130,8138],{},[383,8131,8132],{},[3653,8133,8134,8136],{},[386,8135,795],{},[621,8137,7962],{},[390,8139,8140],{"encoding":392},"\\pi^*",[367,8142,8144],{"className":8143,"ariaHidden":397},[396],[367,8145,8147,8151],{"className":8146},[401],[367,8148],{"className":8149,"style":8150},[405],"height:0.6887em;",[367,8152,8154,8157],{"className":8153},[410],[367,8155,795],{"className":8156,"style":811},[410,411],[367,8158,8160],{"className":8159},[661],[367,8161,8163],{"className":8162},[665],[367,8164,8166],{"className":8165},[670],[367,8167,8169],{"className":8168,"style":8150},[674],[367,8170,8171,8174],{"style":4176},[367,8172],{"className":8173,"style":683},[682],[367,8175,8177],{"className":8176},[687,688,689,690],[367,8178,7962],{"className":8179},[3136,690]," 就是",[315,8182,8183],{},"最优策略（optimal policy）",[357,8185,6974],{"id":6974},[212,8187,8188,8189,8192,8193,8196],{},"知道一个状态或状态-动作对的",[315,8190,8191],{},"价值（value）"," 常常很有用。所谓价值，意思是：如果从该状态或状态-动作对出发，之后一直按某个特定策略行动，所能得到的期望回报。",[315,8194,8195],{},"价值函数（value function）"," 以这样或那样的方式出现在几乎每一个 RL 算法里。",[212,8198,8199],{},"这里有四个值得注意的主要函数。",[8201,8202,8203,8627,9172,9621],"ol",{},[236,8204,8205,365,8208,8287,8288,8316,8317,8345,8346],{},[315,8206,8207],{},"同策略价值函数（on-policy value function）",[367,8209,8211,8236],{"className":8210},[370],[367,8212,8214],{"className":8213},[374],[376,8215,8216],{"xmlns":378},[380,8217,8218,8233],{},[383,8219,8220,8227,8229,8231],{},[3653,8221,8222,8225],{},[386,8223,8224],{},"V",[386,8226,795],{},[621,8228,629],{"stretchy":628},[386,8230,388],{},[621,8232,638],{"stretchy":628},[390,8234,8235],{"encoding":392},"V^{\\pi}(s)",[367,8237,8239],{"className":8238,"ariaHidden":397},[396],[367,8240,8242,8245,8278,8281,8284],{"className":8241},[401],[367,8243],{"className":8244,"style":726},[405],[367,8246,8248,8251],{"className":8247},[410],[367,8249,8224],{"className":8250,"style":3132},[410,411],[367,8252,8254],{"className":8253},[661],[367,8255,8257],{"className":8256},[665],[367,8258,8260],{"className":8259},[670],[367,8261,8264],{"className":8262,"style":8263},[674],"height:0.6644em;",[367,8265,8266,8269],{"style":4176},[367,8267],{"className":8268,"style":683},[682],[367,8270,8272],{"className":8271},[687,688,689,690],[367,8273,8275],{"className":8274},[410,690],[367,8276,795],{"className":8277,"style":811},[410,411,690],[367,8279,629],{"className":8280},[733],[367,8282,388],{"className":8283},[410,411],[367,8285,638],{"className":8286},[777],"：从状态 ",[367,8289,8291,8304],{"className":8290},[370],[367,8292,8294],{"className":8293},[374],[376,8295,8296],{"xmlns":378},[380,8297,8298,8302],{},[383,8299,8300],{},[386,8301,388],{},[390,8303,388],{"encoding":392},[367,8305,8307],{"className":8306,"ariaHidden":397},[396],[367,8308,8310,8313],{"className":8309},[401],[367,8311],{"className":8312,"style":406},[405],[367,8314,388],{"className":8315},[410,411]," 出发、之后一直按策略 ",[367,8318,8320,8333],{"className":8319},[370],[367,8321,8323],{"className":8322},[374],[376,8324,8325],{"xmlns":378},[380,8326,8327,8331],{},[383,8328,8329],{},[386,8330,795],{},[390,8332,798],{"encoding":392},[367,8334,8336],{"className":8335,"ariaHidden":397},[396],[367,8337,8339,8342],{"className":8338},[401],[367,8340],{"className":8341,"style":406},[405],[367,8343,795],{"className":8344,"style":811},[410,411]," 行动，能得到的期望回报：",[367,8347,8349],{"className":8348},[598],[367,8350,8352,8420],{"className":8351},[370],[367,8353,8355],{"className":8354},[374],[376,8356,8357],{"xmlns":378,"display":607},[380,8358,8359,8417],{},[383,8360,8361,8367,8369,8371,8373,8375,8391],{},[3653,8362,8363,8365],{},[386,8364,8224],{},[386,8366,795],{},[621,8368,629],{"stretchy":628},[386,8370,388],{},[621,8372,638],{"stretchy":628},[621,8374,623],{},[386,8376,8377],{},[7697,8378,8379,8383],{},[621,8380,8381],{},[386,8382,7704],{"mathvariant":7703},[383,8384,8385,8387,8389],{},[386,8386,4325],{},[621,8388,835],{},[386,8390,795],{},[383,8392,8393,8395,8397,8399,8401,8403,8405,8411,8413,8415],{},[621,8394,2047],{"fence":397},[386,8396,5734],{},[621,8398,629],{"stretchy":628},[386,8400,4325],{},[621,8402,638],{"stretchy":628},[621,8404,846],{},[613,8406,8407,8409],{},[386,8408,388],{},[2932,8410,1083],{},[621,8412,623],{},[386,8414,388],{},[621,8416,2062],{"fence":397},[390,8418,8419],{"encoding":392},"V^{\\pi}(s) = \\underset{\\tau \\sim \\pi}{\\mathbb{E}}\\left[ R(\\tau) \\mid s_0 = s \\right]",[367,8421,8423,8480],{"className":8422,"ariaHidden":397},[396],[367,8424,8426,8429,8462,8465,8468,8471,8474,8477],{"className":8425},[401],[367,8427],{"className":8428,"style":726},[405],[367,8430,8432,8435],{"className":8431},[410],[367,8433,8224],{"className":8434,"style":3132},[410,411],[367,8436,8438],{"className":8437},[661],[367,8439,8441],{"className":8440},[665],[367,8442,8444],{"className":8443},[670],[367,8445,8448],{"className":8446,"style":8447},[674],"height:0.7144em;",[367,8449,8450,8453],{"style":6904},[367,8451],{"className":8452,"style":683},[682],[367,8454,8456],{"className":8455},[687,688,689,690],[367,8457,8459],{"className":8458},[410,690],[367,8460,795],{"className":8461,"style":811},[410,411,690],[367,8463,629],{"className":8464},[733],[367,8466,388],{"className":8467},[410,411],[367,8469,638],{"className":8470},[777],[367,8472],{"className":8473,"style":712},[711],[367,8475,623],{"className":8476},[716],[367,8478],{"className":8479,"style":712},[711],[367,8481,8483,8486,8542,8545],{"className":8482},[401],[367,8484],{"className":8485,"style":7860},[405],[367,8487,8489],{"className":8488},[410],[367,8490,8492],{"className":8491},[1653,3882],[367,8493,8495,8534],{"className":8494},[665,666],[367,8496,8498,8531],{"className":8497},[670],[367,8499,8501,8521],{"className":8500,"style":7876},[674],[367,8502,8503,8506],{"style":7879},[367,8504],{"className":8505,"style":1197},[682],[367,8507,8509],{"className":8508},[687,688,689,690],[367,8510,8512,8515,8518],{"className":8511},[410,690],[367,8513,4325],{"className":8514,"style":4341},[410,411,690],[367,8516,835],{"className":8517},[716,690],[367,8519,795],{"className":8520,"style":811},[410,411,690],[367,8522,8523,8526],{"style":7900},[367,8524],{"className":8525,"style":1197},[682],[367,8527,8528],{},[367,8529,7704],{"className":8530},[1653,7909],[367,8532,698],{"className":8533},[697],[367,8535,8537],{"className":8536},[670],[367,8538,8540],{"className":8539,"style":7919},[674],[367,8541],{},[367,8543],{"className":8544,"style":1773},[711],[367,8546,8548,8551,8554,8557,8560,8563,8566,8569,8572,8612,8615,8618,8621,8624],{"className":8547},[2170],[367,8549,2047],{"className":8550,"style":2178},[733,2177],[367,8552,5734],{"className":8553,"style":5749},[410,411],[367,8555,629],{"className":8556},[733],[367,8558,4325],{"className":8559,"style":4341},[410,411],[367,8561,638],{"className":8562},[777],[367,8564],{"className":8565,"style":712},[711],[367,8567,846],{"className":8568},[716],[367,8570],{"className":8571,"style":712},[711],[367,8573,8575,8578],{"className":8574},[410],[367,8576,388],{"className":8577},[410,411],[367,8579,8581],{"className":8580},[661],[367,8582,8584,8604],{"className":8583},[665,666],[367,8585,8587,8601],{"className":8586},[670],[367,8588,8590],{"className":8589,"style":4456},[674],[367,8591,8592,8595],{"style":678},[367,8593],{"className":8594,"style":683},[682],[367,8596,8598],{"className":8597},[687,688,689,690],[367,8599,1083],{"className":8600},[410,690],[367,8602,698],{"className":8603},[697],[367,8605,8607],{"className":8606},[670],[367,8608,8610],{"className":8609,"style":705},[674],[367,8611],{},[367,8613],{"className":8614,"style":712},[711],[367,8616,623],{"className":8617},[716],[367,8619],{"className":8620,"style":712},[711],[367,8622,388],{"className":8623},[410,411],[367,8625,2062],{"className":8626,"style":2178},[777,2177],[236,8628,8629,365,8632,8287,8723,8751,8752,8780,8781,8345,8809],{},[315,8630,8631],{},"同策略动作价值函数（on-policy action-value function）",[367,8633,8635,8664],{"className":8634},[370],[367,8636,8638],{"className":8637},[374],[376,8639,8640],{"xmlns":378},[380,8641,8642,8661],{},[383,8643,8644,8651,8653,8655,8657,8659],{},[3653,8645,8646,8649],{},[386,8647,8648],{},"Q",[386,8650,795],{},[621,8652,629],{"stretchy":628},[386,8654,388],{},[621,8656,2656],{"separator":397},[386,8658,216],{},[621,8660,638],{"stretchy":628},[390,8662,8663],{"encoding":392},"Q^{\\pi}(s,a)",[367,8665,8667],{"className":8666,"ariaHidden":397},[396],[367,8668,8670,8673,8705,8708,8711,8714,8717,8720],{"className":8669},[401],[367,8671],{"className":8672,"style":726},[405],[367,8674,8676,8679],{"className":8675},[410],[367,8677,8648],{"className":8678},[410,411],[367,8680,8682],{"className":8681},[661],[367,8683,8685],{"className":8684},[665],[367,8686,8688],{"className":8687},[670],[367,8689,8691],{"className":8690,"style":8263},[674],[367,8692,8693,8696],{"style":4176},[367,8694],{"className":8695,"style":683},[682],[367,8697,8699],{"className":8698},[687,688,689,690],[367,8700,8702],{"className":8701},[410,690],[367,8703,795],{"className":8704,"style":811},[410,411,690],[367,8706,629],{"className":8707},[733],[367,8709,388],{"className":8710},[410,411],[367,8712,2656],{"className":8713},[2685],[367,8715],{"className":8716,"style":1773},[711],[367,8718,216],{"className":8719},[410,411],[367,8721,638],{"className":8722},[777],[367,8724,8726,8739],{"className":8725},[370],[367,8727,8729],{"className":8728},[374],[376,8730,8731],{"xmlns":378},[380,8732,8733,8737],{},[383,8734,8735],{},[386,8736,388],{},[390,8738,388],{"encoding":392},[367,8740,8742],{"className":8741,"ariaHidden":397},[396],[367,8743,8745,8748],{"className":8744},[401],[367,8746],{"className":8747,"style":406},[405],[367,8749,388],{"className":8750},[410,411]," 出发，先采取一个任意动作 ",[367,8753,8755,8768],{"className":8754},[370],[367,8756,8758],{"className":8757},[374],[376,8759,8760],{"xmlns":378},[380,8761,8762,8766],{},[383,8763,8764],{},[386,8765,216],{},[390,8767,216],{"encoding":392},[367,8769,8771],{"className":8770,"ariaHidden":397},[396],[367,8772,8774,8777],{"className":8773},[401],[367,8775],{"className":8776,"style":406},[405],[367,8778,216],{"className":8779},[410,411],"（它可以不来自策略），之后一直按策略 ",[367,8782,8784,8797],{"className":8783},[370],[367,8785,8787],{"className":8786},[374],[376,8788,8789],{"xmlns":378},[380,8790,8791,8795],{},[383,8792,8793],{},[386,8794,795],{},[390,8796,798],{"encoding":392},[367,8798,8800],{"className":8799,"ariaHidden":397},[396],[367,8801,8803,8806],{"className":8802},[401],[367,8804],{"className":8805,"style":406},[405],[367,8807,795],{"className":8808,"style":811},[410,411],[367,8810,8812],{"className":8811},[598],[367,8813,8815,8899],{"className":8814},[370],[367,8816,8818],{"className":8817},[374],[376,8819,8820],{"xmlns":378,"display":607},[380,8821,8822,8896],{},[383,8823,8824,8830,8832,8834,8836,8838,8840,8842,8858],{},[3653,8825,8826,8828],{},[386,8827,8648],{},[386,8829,795],{},[621,8831,629],{"stretchy":628},[386,8833,388],{},[621,8835,2656],{"separator":397},[386,8837,216],{},[621,8839,638],{"stretchy":628},[621,8841,623],{},[386,8843,8844],{},[7697,8845,8846,8850],{},[621,8847,8848],{},[386,8849,7704],{"mathvariant":7703},[383,8851,8852,8854,8856],{},[386,8853,4325],{},[621,8855,835],{},[386,8857,795],{},[383,8859,8860,8862,8864,8866,8868,8870,8872,8878,8880,8882,8884,8890,8892,8894],{},[621,8861,2047],{"fence":397},[386,8863,5734],{},[621,8865,629],{"stretchy":628},[386,8867,4325],{},[621,8869,638],{"stretchy":628},[621,8871,846],{},[613,8873,8874,8876],{},[386,8875,388],{},[2932,8877,1083],{},[621,8879,623],{},[386,8881,388],{},[621,8883,2656],{"separator":397},[613,8885,8886,8888],{},[386,8887,216],{},[2932,8889,1083],{},[621,8891,623],{},[386,8893,216],{},[621,8895,2062],{"fence":397},[390,8897,8898],{"encoding":392},"Q^{\\pi}(s,a) = \\underset{\\tau \\sim \\pi}{\\mathbb{E}}\\left[ R(\\tau) \\mid s_0 = s, a_0 = a \\right]",[367,8900,8902,8967],{"className":8901,"ariaHidden":397},[396],[367,8903,8905,8908,8940,8943,8946,8949,8952,8955,8958,8961,8964],{"className":8904},[401],[367,8906],{"className":8907,"style":726},[405],[367,8909,8911,8914],{"className":8910},[410],[367,8912,8648],{"className":8913},[410,411],[367,8915,8917],{"className":8916},[661],[367,8918,8920],{"className":8919},[665],[367,8921,8923],{"className":8922},[670],[367,8924,8926],{"className":8925,"style":8447},[674],[367,8927,8928,8931],{"style":6904},[367,8929],{"className":8930,"style":683},[682],[367,8932,8934],{"className":8933},[687,688,689,690],[367,8935,8937],{"className":8936},[410,690],[367,8938,795],{"className":8939,"style":811},[410,411,690],[367,8941,629],{"className":8942},[733],[367,8944,388],{"className":8945},[410,411],[367,8947,2656],{"className":8948},[2685],[367,8950],{"className":8951,"style":1773},[711],[367,8953,216],{"className":8954},[410,411],[367,8956,638],{"className":8957},[777],[367,8959],{"className":8960,"style":712},[711],[367,8962,623],{"className":8963},[716],[367,8965],{"className":8966,"style":712},[711],[367,8968,8970,8973,9029,9032],{"className":8969},[401],[367,8971],{"className":8972,"style":7860},[405],[367,8974,8976],{"className":8975},[410],[367,8977,8979],{"className":8978},[1653,3882],[367,8980,8982,9021],{"className":8981},[665,666],[367,8983,8985,9018],{"className":8984},[670],[367,8986,8988,9008],{"className":8987,"style":7876},[674],[367,8989,8990,8993],{"style":7879},[367,8991],{"className":8992,"style":1197},[682],[367,8994,8996],{"className":8995},[687,688,689,690],[367,8997,8999,9002,9005],{"className":8998},[410,690],[367,9000,4325],{"className":9001,"style":4341},[410,411,690],[367,9003,835],{"className":9004},[716,690],[367,9006,795],{"className":9007,"style":811},[410,411,690],[367,9009,9010,9013],{"style":7900},[367,9011],{"className":9012,"style":1197},[682],[367,9014,9015],{},[367,9016,7704],{"className":9017},[1653,7909],[367,9019,698],{"className":9020},[697],[367,9022,9024],{"className":9023},[670],[367,9025,9027],{"className":9026,"style":7919},[674],[367,9028],{},[367,9030],{"className":9031,"style":1773},[711],[367,9033,9035,9038,9041,9044,9047,9050,9053,9056,9059,9099,9102,9105,9108,9111,9114,9117,9157,9160,9163,9166,9169],{"className":9034},[2170],[367,9036,2047],{"className":9037,"style":2178},[733,2177],[367,9039,5734],{"className":9040,"style":5749},[410,411],[367,9042,629],{"className":9043},[733],[367,9045,4325],{"className":9046,"style":4341},[410,411],[367,9048,638],{"className":9049},[777],[367,9051],{"className":9052,"style":712},[711],[367,9054,846],{"className":9055},[716],[367,9057],{"className":9058,"style":712},[711],[367,9060,9062,9065],{"className":9061},[410],[367,9063,388],{"className":9064},[410,411],[367,9066,9068],{"className":9067},[661],[367,9069,9071,9091],{"className":9070},[665,666],[367,9072,9074,9088],{"className":9073},[670],[367,9075,9077],{"className":9076,"style":4456},[674],[367,9078,9079,9082],{"style":678},[367,9080],{"className":9081,"style":683},[682],[367,9083,9085],{"className":9084},[687,688,689,690],[367,9086,1083],{"className":9087},[410,690],[367,9089,698],{"className":9090},[697],[367,9092,9094],{"className":9093},[670],[367,9095,9097],{"className":9096,"style":705},[674],[367,9098],{},[367,9100],{"className":9101,"style":712},[711],[367,9103,623],{"className":9104},[716],[367,9106],{"className":9107,"style":712},[711],[367,9109,388],{"className":9110},[410,411],[367,9112,2656],{"className":9113},[2685],[367,9115],{"className":9116,"style":1773},[711],[367,9118,9120,9123],{"className":9119},[410],[367,9121,216],{"className":9122},[410,411],[367,9124,9126],{"className":9125},[661],[367,9127,9129,9149],{"className":9128},[665,666],[367,9130,9132,9146],{"className":9131},[670],[367,9133,9135],{"className":9134,"style":4456},[674],[367,9136,9137,9140],{"style":678},[367,9138],{"className":9139,"style":683},[682],[367,9141,9143],{"className":9142},[687,688,689,690],[367,9144,1083],{"className":9145},[410,690],[367,9147,698],{"className":9148},[697],[367,9150,9152],{"className":9151},[670],[367,9153,9155],{"className":9154,"style":705},[674],[367,9156],{},[367,9158],{"className":9159,"style":712},[711],[367,9161,623],{"className":9162},[716],[367,9164],{"className":9165,"style":712},[711],[367,9167,216],{"className":9168},[410,411],[367,9170,2062],{"className":9171,"style":2178},[777,2177],[236,9173,9174,365,9177,8287,9251,9279,9280,9283,9284],{},[315,9175,9176],{},"最优价值函数（optimal value function）",[367,9178,9180,9204],{"className":9179},[370],[367,9181,9183],{"className":9182},[374],[376,9184,9185],{"xmlns":378},[380,9186,9187,9201],{},[383,9188,9189,9195,9197,9199],{},[3653,9190,9191,9193],{},[386,9192,8224],{},[621,9194,7962],{},[621,9196,629],{"stretchy":628},[386,9198,388],{},[621,9200,638],{"stretchy":628},[390,9202,9203],{"encoding":392},"V^*(s)",[367,9205,9207],{"className":9206,"ariaHidden":397},[396],[367,9208,9210,9213,9242,9245,9248],{"className":9209},[401],[367,9211],{"className":9212,"style":726},[405],[367,9214,9216,9219],{"className":9215},[410],[367,9217,8224],{"className":9218,"style":3132},[410,411],[367,9220,9222],{"className":9221},[661],[367,9223,9225],{"className":9224},[665],[367,9226,9228],{"className":9227},[670],[367,9229,9231],{"className":9230,"style":8150},[674],[367,9232,9233,9236],{"style":4176},[367,9234],{"className":9235,"style":683},[682],[367,9237,9239],{"className":9238},[687,688,689,690],[367,9240,7962],{"className":9241},[3136,690],[367,9243,629],{"className":9244},[733],[367,9246,388],{"className":9247},[410,411],[367,9249,638],{"className":9250},[777],[367,9252,9254,9267],{"className":9253},[370],[367,9255,9257],{"className":9256},[374],[376,9258,9259],{"xmlns":378},[380,9260,9261,9265],{},[383,9262,9263],{},[386,9264,388],{},[390,9266,388],{"encoding":392},[367,9268,9270],{"className":9269,"ariaHidden":397},[396],[367,9271,9273,9276],{"className":9272},[401],[367,9274],{"className":9275,"style":406},[405],[367,9277,388],{"className":9278},[410,411]," 出发、之后一直按环境中",[243,9281,9282],{},"最优","的策略行动，能得到的期望回报：",[367,9285,9287],{"className":9286},[598],[367,9288,9290,9368],{"className":9289},[370],[367,9291,9293],{"className":9292},[374],[376,9294,9295],{"xmlns":378,"display":607},[380,9296,9297,9365],{},[383,9298,9299,9305,9307,9309,9311,9313,9323,9339],{},[3653,9300,9301,9303],{},[386,9302,8224],{},[621,9304,7962],{},[621,9306,629],{"stretchy":628},[386,9308,388],{},[621,9310,638],{"stretchy":628},[621,9312,623],{},[7697,9314,9315,9321],{},[383,9316,9317,9319],{},[386,9318,7976],{},[621,9320,1637],{},[386,9322,795],{},[386,9324,9325],{},[7697,9326,9327,9331],{},[621,9328,9329],{},[386,9330,7704],{"mathvariant":7703},[383,9332,9333,9335,9337],{},[386,9334,4325],{},[621,9336,835],{},[386,9338,795],{},[383,9340,9341,9343,9345,9347,9349,9351,9353,9359,9361,9363],{},[621,9342,2047],{"fence":397},[386,9344,5734],{},[621,9346,629],{"stretchy":628},[386,9348,4325],{},[621,9350,638],{"stretchy":628},[621,9352,846],{},[613,9354,9355,9357],{},[386,9356,388],{},[2932,9358,1083],{},[621,9360,623],{},[386,9362,388],{},[621,9364,2062],{"fence":397},[390,9366,9367],{"encoding":392},"V^*(s) = \\max_{\\pi} \\underset{\\tau \\sim \\pi}{\\mathbb{E}}\\left[ R(\\tau) \\mid s_0 = s \\right]",[367,9369,9371,9424],{"className":9370,"ariaHidden":397},[396],[367,9372,9374,9377,9406,9409,9412,9415,9418,9421],{"className":9373},[401],[367,9375],{"className":9376,"style":726},[405],[367,9378,9380,9383],{"className":9379},[410],[367,9381,8224],{"className":9382,"style":3132},[410,411],[367,9384,9386],{"className":9385},[661],[367,9387,9389],{"className":9388},[665],[367,9390,9392],{"className":9391},[670],[367,9393,9395],{"className":9394,"style":8001},[674],[367,9396,9397,9400],{"style":6904},[367,9398],{"className":9399,"style":683},[682],[367,9401,9403],{"className":9402},[687,688,689,690],[367,9404,7962],{"className":9405},[3136,690],[367,9407,629],{"className":9408},[733],[367,9410,388],{"className":9411},[410,411],[367,9413,638],{"className":9414},[777],[367,9416],{"className":9417,"style":712},[711],[367,9419,623],{"className":9420},[716],[367,9422],{"className":9423,"style":712},[711],[367,9425,9427,9430,9477,9480,9536,9539],{"className":9426},[401],[367,9428],{"className":9429,"style":7860},[405],[367,9431,9433],{"className":9432},[1653,3882],[367,9434,9436,9469],{"className":9435},[665,666],[367,9437,9439,9466],{"className":9438},[670],[367,9440,9442,9456],{"className":9441,"style":406},[674],[367,9443,9444,9447],{"style":7879},[367,9445],{"className":9446,"style":1197},[682],[367,9448,9450],{"className":9449},[687,688,689,690],[367,9451,9453],{"className":9452},[410,690],[367,9454,795],{"className":9455,"style":811},[410,411,690],[367,9457,9458,9461],{"style":7900},[367,9459],{"className":9460,"style":1197},[682],[367,9462,9463],{},[367,9464,7976],{"className":9465},[1653],[367,9467,698],{"className":9468},[697],[367,9470,9472],{"className":9471},[670],[367,9473,9475],{"className":9474,"style":7919},[674],[367,9476],{},[367,9478],{"className":9479,"style":1773},[711],[367,9481,9483],{"className":9482},[410],[367,9484,9486],{"className":9485},[1653,3882],[367,9487,9489,9528],{"className":9488},[665,666],[367,9490,9492,9525],{"className":9491},[670],[367,9493,9495,9515],{"className":9494,"style":7876},[674],[367,9496,9497,9500],{"style":7879},[367,9498],{"className":9499,"style":1197},[682],[367,9501,9503],{"className":9502},[687,688,689,690],[367,9504,9506,9509,9512],{"className":9505},[410,690],[367,9507,4325],{"className":9508,"style":4341},[410,411,690],[367,9510,835],{"className":9511},[716,690],[367,9513,795],{"className":9514,"style":811},[410,411,690],[367,9516,9517,9520],{"style":7900},[367,9518],{"className":9519,"style":1197},[682],[367,9521,9522],{},[367,9523,7704],{"className":9524},[1653,7909],[367,9526,698],{"className":9527},[697],[367,9529,9531],{"className":9530},[670],[367,9532,9534],{"className":9533,"style":7919},[674],[367,9535],{},[367,9537],{"className":9538,"style":1773},[711],[367,9540,9542,9545,9548,9551,9554,9557,9560,9563,9566,9606,9609,9612,9615,9618],{"className":9541},[2170],[367,9543,2047],{"className":9544,"style":2178},[733,2177],[367,9546,5734],{"className":9547,"style":5749},[410,411],[367,9549,629],{"className":9550},[733],[367,9552,4325],{"className":9553,"style":4341},[410,411],[367,9555,638],{"className":9556},[777],[367,9558],{"className":9559,"style":712},[711],[367,9561,846],{"className":9562},[716],[367,9564],{"className":9565,"style":712},[711],[367,9567,9569,9572],{"className":9568},[410],[367,9570,388],{"className":9571},[410,411],[367,9573,9575],{"className":9574},[661],[367,9576,9578,9598],{"className":9577},[665,666],[367,9579,9581,9595],{"className":9580},[670],[367,9582,9584],{"className":9583,"style":4456},[674],[367,9585,9586,9589],{"style":678},[367,9587],{"className":9588,"style":683},[682],[367,9590,9592],{"className":9591},[687,688,689,690],[367,9593,1083],{"className":9594},[410,690],[367,9596,698],{"className":9597},[697],[367,9599,9601],{"className":9600},[670],[367,9602,9604],{"className":9603,"style":705},[674],[367,9605],{},[367,9607],{"className":9608,"style":712},[711],[367,9610,623],{"className":9611},[716],[367,9613],{"className":9614,"style":712},[711],[367,9616,388],{"className":9617},[410,411],[367,9619,2062],{"className":9620,"style":2178},[777,2177],[236,9622,9623,365,9626,8287,9713,8751,9741,9769,9770,9283,9772],{},[315,9624,9625],{},"最优动作价值函数（optimal action-value function）",[367,9627,9629,9657],{"className":9628},[370],[367,9630,9632],{"className":9631},[374],[376,9633,9634],{"xmlns":378},[380,9635,9636,9654],{},[383,9637,9638,9644,9646,9648,9650,9652],{},[3653,9639,9640,9642],{},[386,9641,8648],{},[621,9643,7962],{},[621,9645,629],{"stretchy":628},[386,9647,388],{},[621,9649,2656],{"separator":397},[386,9651,216],{},[621,9653,638],{"stretchy":628},[390,9655,9656],{"encoding":392},"Q^*(s,a)",[367,9658,9660],{"className":9659,"ariaHidden":397},[396],[367,9661,9663,9666,9695,9698,9701,9704,9707,9710],{"className":9662},[401],[367,9664],{"className":9665,"style":726},[405],[367,9667,9669,9672],{"className":9668},[410],[367,9670,8648],{"className":9671},[410,411],[367,9673,9675],{"className":9674},[661],[367,9676,9678],{"className":9677},[665],[367,9679,9681],{"className":9680},[670],[367,9682,9684],{"className":9683,"style":8150},[674],[367,9685,9686,9689],{"style":4176},[367,9687],{"className":9688,"style":683},[682],[367,9690,9692],{"className":9691},[687,688,689,690],[367,9693,7962],{"className":9694},[3136,690],[367,9696,629],{"className":9697},[733],[367,9699,388],{"className":9700},[410,411],[367,9702,2656],{"className":9703},[2685],[367,9705],{"className":9706,"style":1773},[711],[367,9708,216],{"className":9709},[410,411],[367,9711,638],{"className":9712},[777],[367,9714,9716,9729],{"className":9715},[370],[367,9717,9719],{"className":9718},[374],[376,9720,9721],{"xmlns":378},[380,9722,9723,9727],{},[383,9724,9725],{},[386,9726,388],{},[390,9728,388],{"encoding":392},[367,9730,9732],{"className":9731,"ariaHidden":397},[396],[367,9733,9735,9738],{"className":9734},[401],[367,9736],{"className":9737,"style":406},[405],[367,9739,388],{"className":9740},[410,411],[367,9742,9744,9757],{"className":9743},[370],[367,9745,9747],{"className":9746},[374],[376,9748,9749],{"xmlns":378},[380,9750,9751,9755],{},[383,9752,9753],{},[386,9754,216],{},[390,9756,216],{"encoding":392},[367,9758,9760],{"className":9759,"ariaHidden":397},[396],[367,9761,9763,9766],{"className":9762},[401],[367,9764],{"className":9765,"style":406},[405],[367,9767,216],{"className":9768},[410,411],"，之后一直按环境中",[243,9771,9282],{},[367,9773,9775],{"className":9774},[598],[367,9776,9778,9872],{"className":9777},[370],[367,9779,9781],{"className":9780},[374],[376,9782,9783],{"xmlns":378,"display":607},[380,9784,9785,9869],{},[383,9786,9787,9793,9795,9797,9799,9801,9803,9805,9815,9831],{},[3653,9788,9789,9791],{},[386,9790,8648],{},[621,9792,7962],{},[621,9794,629],{"stretchy":628},[386,9796,388],{},[621,9798,2656],{"separator":397},[386,9800,216],{},[621,9802,638],{"stretchy":628},[621,9804,623],{},[7697,9806,9807,9813],{},[383,9808,9809,9811],{},[386,9810,7976],{},[621,9812,1637],{},[386,9814,795],{},[386,9816,9817],{},[7697,9818,9819,9823],{},[621,9820,9821],{},[386,9822,7704],{"mathvariant":7703},[383,9824,9825,9827,9829],{},[386,9826,4325],{},[621,9828,835],{},[386,9830,795],{},[383,9832,9833,9835,9837,9839,9841,9843,9845,9851,9853,9855,9857,9863,9865,9867],{},[621,9834,2047],{"fence":397},[386,9836,5734],{},[621,9838,629],{"stretchy":628},[386,9840,4325],{},[621,9842,638],{"stretchy":628},[621,9844,846],{},[613,9846,9847,9849],{},[386,9848,388],{},[2932,9850,1083],{},[621,9852,623],{},[386,9854,388],{},[621,9856,2656],{"separator":397},[613,9858,9859,9861],{},[386,9860,216],{},[2932,9862,1083],{},[621,9864,623],{},[386,9866,216],{},[621,9868,2062],{"fence":397},[390,9870,9871],{"encoding":392},"Q^*(s,a) = \\max_{\\pi} \\underset{\\tau \\sim \\pi}{\\mathbb{E}}\\left[ R(\\tau) \\mid s_0 = s, a_0 = a \\right]",[367,9873,9875,9937],{"className":9874,"ariaHidden":397},[396],[367,9876,9878,9881,9910,9913,9916,9919,9922,9925,9928,9931,9934],{"className":9877},[401],[367,9879],{"className":9880,"style":726},[405],[367,9882,9884,9887],{"className":9883},[410],[367,9885,8648],{"className":9886},[410,411],[367,9888,9890],{"className":9889},[661],[367,9891,9893],{"className":9892},[665],[367,9894,9896],{"className":9895},[670],[367,9897,9899],{"className":9898,"style":8001},[674],[367,9900,9901,9904],{"style":6904},[367,9902],{"className":9903,"style":683},[682],[367,9905,9907],{"className":9906},[687,688,689,690],[367,9908,7962],{"className":9909},[3136,690],[367,9911,629],{"className":9912},[733],[367,9914,388],{"className":9915},[410,411],[367,9917,2656],{"className":9918},[2685],[367,9920],{"className":9921,"style":1773},[711],[367,9923,216],{"className":9924},[410,411],[367,9926,638],{"className":9927},[777],[367,9929],{"className":9930,"style":712},[711],[367,9932,623],{"className":9933},[716],[367,9935],{"className":9936,"style":712},[711],[367,9938,9940,9943,9990,9993,10049,10052],{"className":9939},[401],[367,9941],{"className":9942,"style":7860},[405],[367,9944,9946],{"className":9945},[1653,3882],[367,9947,9949,9982],{"className":9948},[665,666],[367,9950,9952,9979],{"className":9951},[670],[367,9953,9955,9969],{"className":9954,"style":406},[674],[367,9956,9957,9960],{"style":7879},[367,9958],{"className":9959,"style":1197},[682],[367,9961,9963],{"className":9962},[687,688,689,690],[367,9964,9966],{"className":9965},[410,690],[367,9967,795],{"className":9968,"style":811},[410,411,690],[367,9970,9971,9974],{"style":7900},[367,9972],{"className":9973,"style":1197},[682],[367,9975,9976],{},[367,9977,7976],{"className":9978},[1653],[367,9980,698],{"className":9981},[697],[367,9983,9985],{"className":9984},[670],[367,9986,9988],{"className":9987,"style":7919},[674],[367,9989],{},[367,9991],{"className":9992,"style":1773},[711],[367,9994,9996],{"className":9995},[410],[367,9997,9999],{"className":9998},[1653,3882],[367,10000,10002,10041],{"className":10001},[665,666],[367,10003,10005,10038],{"className":10004},[670],[367,10006,10008,10028],{"className":10007,"style":7876},[674],[367,10009,10010,10013],{"style":7879},[367,10011],{"className":10012,"style":1197},[682],[367,10014,10016],{"className":10015},[687,688,689,690],[367,10017,10019,10022,10025],{"className":10018},[410,690],[367,10020,4325],{"className":10021,"style":4341},[410,411,690],[367,10023,835],{"className":10024},[716,690],[367,10026,795],{"className":10027,"style":811},[410,411,690],[367,10029,10030,10033],{"style":7900},[367,10031],{"className":10032,"style":1197},[682],[367,10034,10035],{},[367,10036,7704],{"className":10037},[1653,7909],[367,10039,698],{"className":10040},[697],[367,10042,10044],{"className":10043},[670],[367,10045,10047],{"className":10046,"style":7919},[674],[367,10048],{},[367,10050],{"className":10051,"style":1773},[711],[367,10053,10055,10058,10061,10064,10067,10070,10073,10076,10079,10119,10122,10125,10128,10131,10134,10137,10177,10180,10183,10186,10189],{"className":10054},[2170],[367,10056,2047],{"className":10057,"style":2178},[733,2177],[367,10059,5734],{"className":10060,"style":5749},[410,411],[367,10062,629],{"className":10063},[733],[367,10065,4325],{"className":10066,"style":4341},[410,411],[367,10068,638],{"className":10069},[777],[367,10071],{"className":10072,"style":712},[711],[367,10074,846],{"className":10075},[716],[367,10077],{"className":10078,"style":712},[711],[367,10080,10082,10085],{"className":10081},[410],[367,10083,388],{"className":10084},[410,411],[367,10086,10088],{"className":10087},[661],[367,10089,10091,10111],{"className":10090},[665,666],[367,10092,10094,10108],{"className":10093},[670],[367,10095,10097],{"className":10096,"style":4456},[674],[367,10098,10099,10102],{"style":678},[367,10100],{"className":10101,"style":683},[682],[367,10103,10105],{"className":10104},[687,688,689,690],[367,10106,1083],{"className":10107},[410,690],[367,10109,698],{"className":10110},[697],[367,10112,10114],{"className":10113},[670],[367,10115,10117],{"className":10116,"style":705},[674],[367,10118],{},[367,10120],{"className":10121,"style":712},[711],[367,10123,623],{"className":10124},[716],[367,10126],{"className":10127,"style":712},[711],[367,10129,388],{"className":10130},[410,411],[367,10132,2656],{"className":10133},[2685],[367,10135],{"className":10136,"style":1773},[711],[367,10138,10140,10143],{"className":10139},[410],[367,10141,216],{"className":10142},[410,411],[367,10144,10146],{"className":10145},[661],[367,10147,10149,10169],{"className":10148},[665,666],[367,10150,10152,10166],{"className":10151},[670],[367,10153,10155],{"className":10154,"style":4456},[674],[367,10156,10157,10160],{"style":678},[367,10158],{"className":10159,"style":683},[682],[367,10161,10163],{"className":10162},[687,688,689,690],[367,10164,1083],{"className":10165},[410,690],[367,10167,698],{"className":10168},[697],[367,10170,10172],{"className":10171},[670],[367,10173,10175],{"className":10174,"style":705},[674],[367,10176],{},[367,10178],{"className":10179,"style":712},[711],[367,10181,623],{"className":10182},[716],[367,10184],{"className":10185,"style":712},[711],[367,10187,216],{"className":10188},[410,411],[367,10190,2062],{"className":10191,"style":2178},[777,2177],[208,10193,10194],{"title":462},[212,10195,10196,10197,10200],{},"谈价值函数时，如果没有专门提到对时间的依赖，指的都是期望",[315,10198,10199],{},"无限视界折扣回报","。有限视界不折扣回报的价值函数必须把时间作为参数传入。你能想到为什么吗？提示：时间到了会发生什么？",[208,10202,10203,10206,10459,10462,10676],{"title":462},[212,10204,10205],{},"价值函数与动作价值函数之间有两个经常用到的关键联系：",[367,10207,10209],{"className":10208},[598],[367,10210,10212,10276],{"className":10211},[370],[367,10213,10215],{"className":10214},[374],[376,10216,10217],{"xmlns":378,"display":607},[380,10218,10219,10273],{},[383,10220,10221,10227,10229,10231,10233,10235,10251],{},[3653,10222,10223,10225],{},[386,10224,8224],{},[386,10226,795],{},[621,10228,629],{"stretchy":628},[386,10230,388],{},[621,10232,638],{"stretchy":628},[621,10234,623],{},[386,10236,10237],{},[7697,10238,10239,10243],{},[621,10240,10241],{},[386,10242,7704],{"mathvariant":7703},[383,10244,10245,10247,10249],{},[386,10246,216],{},[621,10248,835],{},[386,10250,795],{},[383,10252,10253,10255,10261,10263,10265,10267,10269,10271],{},[621,10254,2047],{"fence":397},[3653,10256,10257,10259],{},[386,10258,8648],{},[386,10260,795],{},[621,10262,629],{"stretchy":628},[386,10264,388],{},[621,10266,2656],{"separator":397},[386,10268,216],{},[621,10270,638],{"stretchy":628},[621,10272,2062],{"fence":397},[390,10274,10275],{"encoding":392},"V^{\\pi}(s) = \\underset{a\\sim \\pi}{\\mathbb{E}}\\left[ Q^{\\pi}(s,a) \\right]",[367,10277,10279,10335],{"className":10278,"ariaHidden":397},[396],[367,10280,10282,10285,10317,10320,10323,10326,10329,10332],{"className":10281},[401],[367,10283],{"className":10284,"style":726},[405],[367,10286,10288,10291],{"className":10287},[410],[367,10289,8224],{"className":10290,"style":3132},[410,411],[367,10292,10294],{"className":10293},[661],[367,10295,10297],{"className":10296},[665],[367,10298,10300],{"className":10299},[670],[367,10301,10303],{"className":10302,"style":8447},[674],[367,10304,10305,10308],{"style":6904},[367,10306],{"className":10307,"style":683},[682],[367,10309,10311],{"className":10310},[687,688,689,690],[367,10312,10314],{"className":10313},[410,690],[367,10315,795],{"className":10316,"style":811},[410,411,690],[367,10318,629],{"className":10319},[733],[367,10321,388],{"className":10322},[410,411],[367,10324,638],{"className":10325},[777],[367,10327],{"className":10328,"style":712},[711],[367,10330,623],{"className":10331},[716],[367,10333],{"className":10334,"style":712},[711],[367,10336,10338,10341,10397,10400],{"className":10337},[401],[367,10339],{"className":10340,"style":7860},[405],[367,10342,10344],{"className":10343},[410],[367,10345,10347],{"className":10346},[1653,3882],[367,10348,10350,10389],{"className":10349},[665,666],[367,10351,10353,10386],{"className":10352},[670],[367,10354,10356,10376],{"className":10355,"style":7876},[674],[367,10357,10358,10361],{"style":7879},[367,10359],{"className":10360,"style":1197},[682],[367,10362,10364],{"className":10363},[687,688,689,690],[367,10365,10367,10370,10373],{"className":10366},[410,690],[367,10368,216],{"className":10369},[410,411,690],[367,10371,835],{"className":10372},[716,690],[367,10374,795],{"className":10375,"style":811},[410,411,690],[367,10377,10378,10381],{"style":7900},[367,10379],{"className":10380,"style":1197},[682],[367,10382,10383],{},[367,10384,7704],{"className":10385},[1653,7909],[367,10387,698],{"className":10388},[697],[367,10390,10392],{"className":10391},[670],[367,10393,10395],{"className":10394,"style":7919},[674],[367,10396],{},[367,10398],{"className":10399,"style":1773},[711],[367,10401,10403,10406,10438,10441,10444,10447,10450,10453,10456],{"className":10402},[2170],[367,10404,2047],{"className":10405,"style":2178},[733,2177],[367,10407,10409,10412],{"className":10408},[410],[367,10410,8648],{"className":10411},[410,411],[367,10413,10415],{"className":10414},[661],[367,10416,10418],{"className":10417},[665],[367,10419,10421],{"className":10420},[670],[367,10422,10424],{"className":10423,"style":8447},[674],[367,10425,10426,10429],{"style":6904},[367,10427],{"className":10428,"style":683},[682],[367,10430,10432],{"className":10431},[687,688,689,690],[367,10433,10435],{"className":10434},[410,690],[367,10436,795],{"className":10437,"style":811},[410,411,690],[367,10439,629],{"className":10440},[733],[367,10442,388],{"className":10443},[410,411],[367,10445,2656],{"className":10446},[2685],[367,10448],{"className":10449,"style":1773},[711],[367,10451,216],{"className":10452},[410,411],[367,10454,638],{"className":10455},[777],[367,10457,2062],{"className":10458,"style":2178},[777,2177],[212,10460,10461],{},"以及",[367,10463,10465],{"className":10464},[598],[367,10466,10468,10520],{"className":10467},[370],[367,10469,10471],{"className":10470},[374],[376,10472,10473],{"xmlns":378,"display":607},[380,10474,10475,10517],{},[383,10476,10477,10483,10485,10487,10489,10491,10501,10507,10509,10511,10513,10515],{},[3653,10478,10479,10481],{},[386,10480,8224],{},[621,10482,7962],{},[621,10484,629],{"stretchy":628},[386,10486,388],{},[621,10488,638],{"stretchy":628},[621,10490,623],{},[7697,10492,10493,10499],{},[383,10494,10495,10497],{},[386,10496,7976],{},[621,10498,1637],{},[386,10500,216],{},[3653,10502,10503,10505],{},[386,10504,8648],{},[621,10506,7962],{},[621,10508,629],{"stretchy":628},[386,10510,388],{},[621,10512,2656],{"separator":397},[386,10514,216],{},[621,10516,638],{"stretchy":628},[390,10518,10519],{"encoding":392},"V^*(s) = \\max_a Q^*(s,a)",[367,10521,10523,10576],{"className":10522,"ariaHidden":397},[396],[367,10524,10526,10529,10558,10561,10564,10567,10570,10573],{"className":10525},[401],[367,10527],{"className":10528,"style":726},[405],[367,10530,10532,10535],{"className":10531},[410],[367,10533,8224],{"className":10534,"style":3132},[410,411],[367,10536,10538],{"className":10537},[661],[367,10539,10541],{"className":10540},[665],[367,10542,10544],{"className":10543},[670],[367,10545,10547],{"className":10546,"style":8001},[674],[367,10548,10549,10552],{"style":6904},[367,10550],{"className":10551,"style":683},[682],[367,10553,10555],{"className":10554},[687,688,689,690],[367,10556,7962],{"className":10557},[3136,690],[367,10559,629],{"className":10560},[733],[367,10562,388],{"className":10563},[410,411],[367,10565,638],{"className":10566},[777],[367,10568],{"className":10569,"style":712},[711],[367,10571,623],{"className":10572},[716],[367,10574],{"className":10575,"style":712},[711],[367,10577,10579,10582,10626,10629,10658,10661,10664,10667,10670,10673],{"className":10578},[401],[367,10580],{"className":10581,"style":7860},[405],[367,10583,10585],{"className":10584},[1653,3882],[367,10586,10588,10618],{"className":10587},[665,666],[367,10589,10591,10615],{"className":10590},[670],[367,10592,10594,10605],{"className":10593,"style":406},[674],[367,10595,10596,10599],{"style":7879},[367,10597],{"className":10598,"style":1197},[682],[367,10600,10602],{"className":10601},[687,688,689,690],[367,10603,216],{"className":10604},[410,411,690],[367,10606,10607,10610],{"style":7900},[367,10608],{"className":10609,"style":1197},[682],[367,10611,10612],{},[367,10613,7976],{"className":10614},[1653],[367,10616,698],{"className":10617},[697],[367,10619,10621],{"className":10620},[670],[367,10622,10624],{"className":10623,"style":7919},[674],[367,10625],{},[367,10627],{"className":10628,"style":1773},[711],[367,10630,10632,10635],{"className":10631},[410],[367,10633,8648],{"className":10634},[410,411],[367,10636,10638],{"className":10637},[661],[367,10639,10641],{"className":10640},[665],[367,10642,10644],{"className":10643},[670],[367,10645,10647],{"className":10646,"style":8001},[674],[367,10648,10649,10652],{"style":6904},[367,10650],{"className":10651,"style":683},[682],[367,10653,10655],{"className":10654},[687,688,689,690],[367,10656,7962],{"className":10657},[3136,690],[367,10659,629],{"className":10660},[733],[367,10662,388],{"className":10663},[410,411],[367,10665,2656],{"className":10666},[2685],[367,10668],{"className":10669,"style":1773},[711],[367,10671,216],{"className":10672},[410,411],[367,10674,638],{"className":10675},[777],[212,10677,10678],{},"这些关系可以直接从刚才给出的定义推出来：你能证明吗？",[357,10680,10682],{"id":10681},"最优-q-函数与最优动作","最优 Q 函数与最优动作",[212,10684,10685,10686,10772,10773,10859,10860,10888,10889,10917],{},"最优动作价值函数 ",[367,10687,10689,10716],{"className":10688},[370],[367,10690,10692],{"className":10691},[374],[376,10693,10694],{"xmlns":378},[380,10695,10696,10714],{},[383,10697,10698,10704,10706,10708,10710,10712],{},[3653,10699,10700,10702],{},[386,10701,8648],{},[621,10703,7962],{},[621,10705,629],{"stretchy":628},[386,10707,388],{},[621,10709,2656],{"separator":397},[386,10711,216],{},[621,10713,638],{"stretchy":628},[390,10715,9656],{"encoding":392},[367,10717,10719],{"className":10718,"ariaHidden":397},[396],[367,10720,10722,10725,10754,10757,10760,10763,10766,10769],{"className":10721},[401],[367,10723],{"className":10724,"style":726},[405],[367,10726,10728,10731],{"className":10727},[410],[367,10729,8648],{"className":10730},[410,411],[367,10732,10734],{"className":10733},[661],[367,10735,10737],{"className":10736},[665],[367,10738,10740],{"className":10739},[670],[367,10741,10743],{"className":10742,"style":8150},[674],[367,10744,10745,10748],{"style":4176},[367,10746],{"className":10747,"style":683},[682],[367,10749,10751],{"className":10750},[687,688,689,690],[367,10752,7962],{"className":10753},[3136,690],[367,10755,629],{"className":10756},[733],[367,10758,388],{"className":10759},[410,411],[367,10761,2656],{"className":10762},[2685],[367,10764],{"className":10765,"style":1773},[711],[367,10767,216],{"className":10768},[410,411],[367,10770,638],{"className":10771},[777]," 与最优策略所选的动作之间有一个重要联系。按定义，",[367,10774,10776,10803],{"className":10775},[370],[367,10777,10779],{"className":10778},[374],[376,10780,10781],{"xmlns":378},[380,10782,10783,10801],{},[383,10784,10785,10791,10793,10795,10797,10799],{},[3653,10786,10787,10789],{},[386,10788,8648],{},[621,10790,7962],{},[621,10792,629],{"stretchy":628},[386,10794,388],{},[621,10796,2656],{"separator":397},[386,10798,216],{},[621,10800,638],{"stretchy":628},[390,10802,9656],{"encoding":392},[367,10804,10806],{"className":10805,"ariaHidden":397},[396],[367,10807,10809,10812,10841,10844,10847,10850,10853,10856],{"className":10808},[401],[367,10810],{"className":10811,"style":726},[405],[367,10813,10815,10818],{"className":10814},[410],[367,10816,8648],{"className":10817},[410,411],[367,10819,10821],{"className":10820},[661],[367,10822,10824],{"className":10823},[665],[367,10825,10827],{"className":10826},[670],[367,10828,10830],{"className":10829,"style":8150},[674],[367,10831,10832,10835],{"style":4176},[367,10833],{"className":10834,"style":683},[682],[367,10836,10838],{"className":10837},[687,688,689,690],[367,10839,7962],{"className":10840},[3136,690],[367,10842,629],{"className":10843},[733],[367,10845,388],{"className":10846},[410,411],[367,10848,2656],{"className":10849},[2685],[367,10851],{"className":10852,"style":1773},[711],[367,10854,216],{"className":10855},[410,411],[367,10857,638],{"className":10858},[777]," 给出的是：从状态 ",[367,10861,10863,10876],{"className":10862},[370],[367,10864,10866],{"className":10865},[374],[376,10867,10868],{"xmlns":378},[380,10869,10870,10874],{},[383,10871,10872],{},[386,10873,388],{},[390,10875,388],{"encoding":392},[367,10877,10879],{"className":10878,"ariaHidden":397},[396],[367,10880,10882,10885],{"className":10881},[401],[367,10883],{"className":10884,"style":406},[405],[367,10886,388],{"className":10887},[410,411]," 出发、先采取（任意的）动作 ",[367,10890,10892,10905],{"className":10891},[370],[367,10893,10895],{"className":10894},[374],[376,10896,10897],{"xmlns":378},[380,10898,10899,10903],{},[383,10900,10901],{},[386,10902,216],{},[390,10904,216],{"encoding":392},[367,10906,10908],{"className":10907,"ariaHidden":397},[396],[367,10909,10911,10914],{"className":10910},[401],[367,10912],{"className":10913,"style":406},[405],[367,10915,216],{"className":10916},[410,411],"、之后一直按最优策略行动时的期望回报。",[212,10919,10920,10921,10949,10950,10978,10979,11039,11040,594],{},"状态 ",[367,10922,10924,10937],{"className":10923},[370],[367,10925,10927],{"className":10926},[374],[376,10928,10929],{"xmlns":378},[380,10930,10931,10935],{},[383,10932,10933],{},[386,10934,388],{},[390,10936,388],{"encoding":392},[367,10938,10940],{"className":10939,"ariaHidden":397},[396],[367,10941,10943,10946],{"className":10942},[401],[367,10944],{"className":10945,"style":406},[405],[367,10947,388],{"className":10948},[410,411]," 处的最优策略，会选择使「从 ",[367,10951,10953,10966],{"className":10952},[370],[367,10954,10956],{"className":10955},[374],[376,10957,10958],{"xmlns":378},[380,10959,10960,10964],{},[383,10961,10962],{},[386,10963,388],{},[390,10965,388],{"encoding":392},[367,10967,10969],{"className":10968,"ariaHidden":397},[396],[367,10970,10972,10975],{"className":10971},[401],[367,10973],{"className":10974,"style":406},[405],[367,10976,388],{"className":10977},[410,411]," 出发的期望回报」最大的那个动作。因此，如果我们手里有 ",[367,10980,10982,11000],{"className":10981},[370],[367,10983,10985],{"className":10984},[374],[376,10986,10987],{"xmlns":378},[380,10988,10989,10997],{},[383,10990,10991],{},[3653,10992,10993,10995],{},[386,10994,8648],{},[621,10996,7962],{},[390,10998,10999],{"encoding":392},"Q^*",[367,11001,11003],{"className":11002,"ariaHidden":397},[396],[367,11004,11006,11010],{"className":11005},[401],[367,11007],{"className":11008,"style":11009},[405],"height:0.8831em;vertical-align:-0.1944em;",[367,11011,11013,11016],{"className":11012},[410],[367,11014,8648],{"className":11015},[410,411],[367,11017,11019],{"className":11018},[661],[367,11020,11022],{"className":11021},[665],[367,11023,11025],{"className":11024},[670],[367,11026,11028],{"className":11027,"style":8150},[674],[367,11029,11030,11033],{"style":4176},[367,11031],{"className":11032,"style":683},[682],[367,11034,11036],{"className":11035},[687,688,689,690],[367,11037,7962],{"className":11038},[3136,690],"，就可以直接得到最优动作 ",[367,11041,11043,11067],{"className":11042},[370],[367,11044,11046],{"className":11045},[374],[376,11047,11048],{"xmlns":378},[380,11049,11050,11064],{},[383,11051,11052,11058,11060,11062],{},[3653,11053,11054,11056],{},[386,11055,216],{},[621,11057,7962],{},[621,11059,629],{"stretchy":628},[386,11061,388],{},[621,11063,638],{"stretchy":628},[390,11065,11066],{"encoding":392},"a^*(s)",[367,11068,11070],{"className":11069,"ariaHidden":397},[396],[367,11071,11073,11076,11105,11108,11111],{"className":11072},[401],[367,11074],{"className":11075,"style":726},[405],[367,11077,11079,11082],{"className":11078},[410],[367,11080,216],{"className":11081},[410,411],[367,11083,11085],{"className":11084},[661],[367,11086,11088],{"className":11087},[665],[367,11089,11091],{"className":11090},[670],[367,11092,11094],{"className":11093,"style":8150},[674],[367,11095,11096,11099],{"style":4176},[367,11097],{"className":11098,"style":683},[682],[367,11100,11102],{"className":11101},[687,688,689,690],[367,11103,7962],{"className":11104},[3136,690],[367,11106,629],{"className":11107},[733],[367,11109,388],{"className":11110},[410,411],[367,11112,638],{"className":11113},[777],[367,11115,11117],{"className":11116},[598],[367,11118,11120,11176],{"className":11119},[370],[367,11121,11123],{"className":11122},[374],[376,11124,11125],{"xmlns":378,"display":607},[380,11126,11127,11173],{},[383,11128,11129,11135,11137,11139,11141,11143,11145,11147,11157,11163,11165,11167,11169,11171],{},[3653,11130,11131,11133],{},[386,11132,216],{},[621,11134,7962],{},[621,11136,629],{"stretchy":628},[386,11138,388],{},[621,11140,638],{"stretchy":628},[621,11142,623],{},[386,11144,7967],{},[621,11146,1637],{},[7697,11148,11149,11155],{},[383,11150,11151,11153],{},[386,11152,7976],{},[621,11154,1637],{},[386,11156,216],{},[3653,11158,11159,11161],{},[386,11160,8648],{},[621,11162,7962],{},[621,11164,629],{"stretchy":628},[386,11166,388],{},[621,11168,2656],{"separator":397},[386,11170,216],{},[621,11172,638],{"stretchy":628},[390,11174,11175],{"encoding":392},"a^*(s) = \\arg \\max_a Q^*(s,a)",[367,11177,11179,11232],{"className":11178,"ariaHidden":397},[396],[367,11180,11182,11185,11214,11217,11220,11223,11226,11229],{"className":11181},[401],[367,11183],{"className":11184,"style":726},[405],[367,11186,11188,11191],{"className":11187},[410],[367,11189,216],{"className":11190},[410,411],[367,11192,11194],{"className":11193},[661],[367,11195,11197],{"className":11196},[665],[367,11198,11200],{"className":11199},[670],[367,11201,11203],{"className":11202,"style":8001},[674],[367,11204,11205,11208],{"style":6904},[367,11206],{"className":11207,"style":683},[682],[367,11209,11211],{"className":11210},[687,688,689,690],[367,11212,7962],{"className":11213},[3136,690],[367,11215,629],{"className":11216},[733],[367,11218,388],{"className":11219},[410,411],[367,11221,638],{"className":11222},[777],[367,11224],{"className":11225,"style":712},[711],[367,11227,623],{"className":11228},[716],[367,11230],{"className":11231,"style":712},[711],[367,11233,11235,11238,11243,11246,11290,11293,11322,11325,11328,11331,11334,11337],{"className":11234},[401],[367,11236],{"className":11237,"style":7860},[405],[367,11239,8049,11241],{"className":11240},[1653],[367,11242,1769],{"style":1768},[367,11244],{"className":11245,"style":1773},[711],[367,11247,11249],{"className":11248},[1653,3882],[367,11250,11252,11282],{"className":11251},[665,666],[367,11253,11255,11279],{"className":11254},[670],[367,11256,11258,11269],{"className":11257,"style":406},[674],[367,11259,11260,11263],{"style":7879},[367,11261],{"className":11262,"style":1197},[682],[367,11264,11266],{"className":11265},[687,688,689,690],[367,11267,216],{"className":11268},[410,411,690],[367,11270,11271,11274],{"style":7900},[367,11272],{"className":11273,"style":1197},[682],[367,11275,11276],{},[367,11277,7976],{"className":11278},[1653],[367,11280,698],{"className":11281},[697],[367,11283,11285],{"className":11284},[670],[367,11286,11288],{"className":11287,"style":7919},[674],[367,11289],{},[367,11291],{"className":11292,"style":1773},[711],[367,11294,11296,11299],{"className":11295},[410],[367,11297,8648],{"className":11298},[410,411],[367,11300,11302],{"className":11301},[661],[367,11303,11305],{"className":11304},[665],[367,11306,11308],{"className":11307},[670],[367,11309,11311],{"className":11310,"style":8001},[674],[367,11312,11313,11316],{"style":6904},[367,11314],{"className":11315,"style":683},[682],[367,11317,11319],{"className":11318},[687,688,689,690],[367,11320,7962],{"className":11321},[3136,690],[367,11323,629],{"className":11324},[733],[367,11326,388],{"className":11327},[410,411],[367,11329,2656],{"className":11330},[2685],[367,11332],{"className":11333,"style":1773},[711],[367,11335,216],{"className":11336},[410,411],[367,11338,638],{"className":11339},[777],[212,11341,11342,11343,11429],{},"注意：可能有多个动作同时使 ",[367,11344,11346,11373],{"className":11345},[370],[367,11347,11349],{"className":11348},[374],[376,11350,11351],{"xmlns":378},[380,11352,11353,11371],{},[383,11354,11355,11361,11363,11365,11367,11369],{},[3653,11356,11357,11359],{},[386,11358,8648],{},[621,11360,7962],{},[621,11362,629],{"stretchy":628},[386,11364,388],{},[621,11366,2656],{"separator":397},[386,11368,216],{},[621,11370,638],{"stretchy":628},[390,11372,9656],{"encoding":392},[367,11374,11376],{"className":11375,"ariaHidden":397},[396],[367,11377,11379,11382,11411,11414,11417,11420,11423,11426],{"className":11378},[401],[367,11380],{"className":11381,"style":726},[405],[367,11383,11385,11388],{"className":11384},[410],[367,11386,8648],{"className":11387},[410,411],[367,11389,11391],{"className":11390},[661],[367,11392,11394],{"className":11393},[665],[367,11395,11397],{"className":11396},[670],[367,11398,11400],{"className":11399,"style":8150},[674],[367,11401,11402,11405],{"style":4176},[367,11403],{"className":11404,"style":683},[682],[367,11406,11408],{"className":11407},[687,688,689,690],[367,11409,7962],{"className":11410},[3136,690],[367,11412,629],{"className":11413},[733],[367,11415,388],{"className":11416},[410,411],[367,11418,2656],{"className":11419},[2685],[367,11421],{"className":11422,"style":1773},[711],[367,11424,216],{"className":11425},[410,411],[367,11427,638],{"className":11428},[777]," 最大，此时它们全都是最优的，最优策略可以随机选择其中任何一个。但总存在一个确定型地选择动作的最优策略。",[357,11431,11432],{"id":11432},"贝尔曼方程",[212,11434,11435,11436,11439],{},"四个价值函数全都满足一类特殊的自洽方程，称为",[315,11437,11438],{},"贝尔曼方程（Bellman equations）","。贝尔曼方程背后的基本想法是：",[11441,11442,11443],"blockquote",{},[212,11444,11445],{},"你出发点的价值，等于你期望从待在那里得到的奖励，加上你下一落脚点的价值。",[212,11447,11448],{},"同策略价值函数的贝尔曼方程为：",[367,11450,11452],{"className":11451},[598],[367,11453,11455,11709],{"className":11454},[370],[367,11456,11458],{"className":11457},[374],[376,11459,11460],{"xmlns":378,"display":607},[380,11461,11462,11706],{},[1069,11463,11464,11580],{"rowspacing":1071,"columnalign":1072,"columnspacing":1073},[1075,11465,11466,11484],{},[1078,11467,11468],{},[1081,11469,11470],{"scriptlevel":1083,"displaystyle":397},[383,11471,11472,11478,11480,11482],{},[3653,11473,11474,11476],{},[386,11475,8224],{},[386,11477,795],{},[621,11479,629],{"stretchy":628},[386,11481,388],{},[621,11483,638],{"stretchy":628},[1078,11485,11486],{},[1081,11487,11488],{"scriptlevel":1083,"displaystyle":397},[383,11489,11490,11492,11494,11542],{},[383,11491],{},[621,11493,623],{},[386,11495,11496],{},[7697,11497,11498,11502],{},[621,11499,11500],{},[386,11501,7704],{"mathvariant":7703},[1081,11503,11504],{"scriptlevel":3603},[1069,11505,11509,11523],{"rowspacing":11506,"columnalign":11507,"columnspacing":11508},"0.1em","center","1em",[1075,11510,11511],{},[1078,11512,11513],{},[1081,11514,11515],{"scriptlevel":3603,"displaystyle":628},[383,11516,11517,11519,11521],{},[386,11518,216],{},[621,11520,835],{},[386,11522,795],{},[1075,11524,11525],{},[1078,11526,11527],{},[1081,11528,11529],{"scriptlevel":3603,"displaystyle":628},[383,11530,11531,11538,11540],{},[3653,11532,11533,11535],{},[386,11534,388],{},[621,11536,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},"′",[621,11539,835],{},[386,11541,1895],{},[383,11543,11544,11546,11548,11550,11552,11554,11556,11558,11560,11562,11568,11570,11576,11578],{},[621,11545,2047],{"fence":397},[386,11547,5770],{},[621,11549,629],{"stretchy":628},[386,11551,388],{},[621,11553,2656],{"separator":397},[386,11555,216],{},[621,11557,638],{"stretchy":628},[621,11559,3029],{},[386,11561,6657],{},[3653,11563,11564,11566],{},[386,11565,8224],{},[386,11567,795],{},[621,11569,629],{"stretchy":628},[3653,11571,11572,11574],{},[386,11573,388],{},[621,11575,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,11577,638],{"stretchy":628},[621,11579,2062],{"fence":397},[1075,11581,11582,11604],{},[1078,11583,11584],{},[1081,11585,11586],{"scriptlevel":1083,"displaystyle":397},[383,11587,11588,11594,11596,11598,11600,11602],{},[3653,11589,11590,11592],{},[386,11591,8648],{},[386,11593,795],{},[621,11595,629],{"stretchy":628},[386,11597,388],{},[621,11599,2656],{"separator":397},[386,11601,216],{},[621,11603,638],{"stretchy":628},[1078,11605,11606],{},[1081,11607,11608],{"scriptlevel":1083,"displaystyle":397},[383,11609,11610,11612,11614,11634],{},[383,11611],{},[621,11613,623],{},[386,11615,11616],{},[7697,11617,11618,11622],{},[621,11619,11620],{},[386,11621,7704],{"mathvariant":7703},[383,11623,11624,11630,11632],{},[3653,11625,11626,11628],{},[386,11627,388],{},[621,11629,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,11631,835],{},[386,11633,1895],{},[383,11635,11636,11638,11640,11642,11644,11646,11648,11650,11652,11654,11674,11704],{},[621,11637,2047],{"fence":397},[386,11639,5770],{},[621,11641,629],{"stretchy":628},[386,11643,388],{},[621,11645,2656],{"separator":397},[386,11647,216],{},[621,11649,638],{"stretchy":628},[621,11651,3029],{},[386,11653,6657],{},[386,11655,11656],{},[7697,11657,11658,11662],{},[621,11659,11660],{},[386,11661,7704],{"mathvariant":7703},[383,11663,11664,11670,11672],{},[3653,11665,11666,11668],{},[386,11667,216],{},[621,11669,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,11671,835],{},[386,11673,795],{},[383,11675,11676,11678,11684,11686,11692,11694,11700,11702],{},[621,11677,2047],{"fence":397},[3653,11679,11680,11682],{},[386,11681,8648],{},[386,11683,795],{},[621,11685,629],{"stretchy":628},[3653,11687,11688,11690],{},[386,11689,388],{},[621,11691,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,11693,2656],{"separator":397},[3653,11695,11696,11698],{},[386,11697,216],{},[621,11699,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,11701,638],{"stretchy":628},[621,11703,2062],{"fence":397},[621,11705,2062],{"fence":397},[390,11707,11708],{"encoding":392},"\\begin{aligned}\nV^{\\pi}(s) &= \\underset{\\substack{a \\sim \\pi \\\\ s'\\sim P}}{\\mathbb{E}}\\left[r(s,a) + \\gamma V^{\\pi}(s')\\right] \\\\\nQ^{\\pi}(s,a) &= \\underset{s'\\sim P}{\\mathbb{E}}\\left[r(s,a) + \\gamma \\underset{a'\\sim \\pi}{\\mathbb{E}}\\left[Q^{\\pi}(s',a')\\right]\\right]\n\\end{aligned}",[367,11710,11712],{"className":11711,"ariaHidden":397},[396],[367,11713,11715,11719],{"className":11714},[401],[367,11716],{"className":11717,"style":11718},[405],"height:4.4332em;vertical-align:-1.9666em;",[367,11720,11722],{"className":11721},[410],[367,11723,11725,11860],{"className":11724},[1069],[367,11726,11728],{"className":11727},[1180],[367,11729,11731,11851],{"className":11730},[665,666],[367,11732,11734,11848],{"className":11733},[670],[367,11735,11738,11789],{"className":11736,"style":11737},[674],"height:2.4666em;",[367,11739,11741,11745],{"style":11740},"top:-4.7766em;",[367,11742],{"className":11743,"style":11744},[682],"height:3.15em;",[367,11746,11748,11780,11783,11786],{"className":11747},[410],[367,11749,11751,11754],{"className":11750},[410],[367,11752,8224],{"className":11753,"style":3132},[410,411],[367,11755,11757],{"className":11756},[661],[367,11758,11760],{"className":11759},[665],[367,11761,11763],{"className":11762},[670],[367,11764,11766],{"className":11765,"style":8447},[674],[367,11767,11768,11771],{"style":6904},[367,11769],{"className":11770,"style":683},[682],[367,11772,11774],{"className":11773},[687,688,689,690],[367,11775,11777],{"className":11776},[410,690],[367,11778,795],{"className":11779,"style":811},[410,411,690],[367,11781,629],{"className":11782},[733],[367,11784,388],{"className":11785},[410,411],[367,11787,638],{"className":11788},[777],[367,11790,11792,11795],{"style":11791},"top:-1.9277em;",[367,11793],{"className":11794,"style":11744},[682],[367,11796,11798,11830,11833,11836,11839,11842,11845],{"className":11797},[410],[367,11799,11801,11804],{"className":11800},[410],[367,11802,8648],{"className":11803},[410,411],[367,11805,11807],{"className":11806},[661],[367,11808,11810],{"className":11809},[665],[367,11811,11813],{"className":11812},[670],[367,11814,11816],{"className":11815,"style":8447},[674],[367,11817,11818,11821],{"style":6904},[367,11819],{"className":11820,"style":683},[682],[367,11822,11824],{"className":11823},[687,688,689,690],[367,11825,11827],{"className":11826},[410,690],[367,11828,795],{"className":11829,"style":811},[410,411,690],[367,11831,629],{"className":11832},[733],[367,11834,388],{"className":11835},[410,411],[367,11837,2656],{"className":11838},[2685],[367,11840],{"className":11841,"style":1773},[711],[367,11843,216],{"className":11844},[410,411],[367,11846,638],{"className":11847},[777],[367,11849,698],{"className":11850},[697],[367,11852,11854],{"className":11853},[670],[367,11855,11858],{"className":11856,"style":11857},[674],"height:1.9666em;",[367,11859],{},[367,11861,11863],{"className":11862},[1305],[367,11864,11866,12531],{"className":11865},[665,666],[367,11867,11869,12528],{"className":11868},[670],[367,11870,11872,12160],{"className":11871,"style":11737},[674],[367,11873,11874,11877],{"style":11740},[367,11875],{"className":11876,"style":11744},[682],[367,11878,11880,11883,11886,11889,11892,12044,12047],{"className":11879},[410],[367,11881],{"className":11882},[410],[367,11884],{"className":11885,"style":712},[711],[367,11887,623],{"className":11888},[716],[367,11890],{"className":11891,"style":712},[711],[367,11893,11895],{"className":11894},[410],[367,11896,11898],{"className":11897},[1653,3882],[367,11899,11901,12035],{"className":11900},[665,666],[367,11902,11904,12032],{"className":11903},[670],[367,11905,11907,12022],{"className":11906,"style":7876},[674],[367,11908,11910,11913],{"style":11909},"top:-2.0926em;margin-left:0em;",[367,11911],{"className":11912,"style":1197},[682],[367,11914,11916],{"className":11915},[687,688,689,690],[367,11917,11919],{"className":11918},[410,690],[367,11920,11922],{"className":11921},[410,690],[367,11923,11925],{"className":11924},[1069],[367,11926,11929],{"className":11927},[11928],"col-align-c",[367,11930,11932,12013],{"className":11931},[665,666],[367,11933,11935,12010],{"className":11934},[670],[367,11936,11939,11958],{"className":11937,"style":11938},[674],"height:1.0592em;",[367,11940,11942,11946],{"style":11941},"top:-3.4565em;",[367,11943],{"className":11944,"style":11945},[682],"height:2.8278em;",[367,11947,11949,11952,11955],{"className":11948},[410,690],[367,11950,216],{"className":11951},[410,411,690],[367,11953,835],{"className":11954},[716,690],[367,11956,795],{"className":11957,"style":811},[410,411,690],[367,11959,11961,11964],{"style":11960},"top:-2.4486em;",[367,11962],{"className":11963,"style":11945},[682],[367,11965,11967,12004,12007],{"className":11966},[410,690],[367,11968,11970,11973],{"className":11969},[410,690],[367,11971,388],{"className":11972},[410,411,690],[367,11974,11976],{"className":11975},[661],[367,11977,11979],{"className":11978},[665],[367,11980,11982],{"className":11981},[670],[367,11983,11986],{"className":11984,"style":11985},[674],"height:0.8278em;",[367,11987,11989,11993],{"style":11988},"top:-2.931em;margin-right:0.0714em;",[367,11990],{"className":11991,"style":11992},[682],"height:2.5em;",[367,11994,11998],{"className":11995},[687,11996,11997,690],"reset-size3","size1",[367,11999,12001],{"className":12000},[410,690],[367,12002,11537],{"className":12003},[410,690],[367,12005,835],{"className":12006},[716,690],[367,12008,1895],{"className":12009,"style":1922},[410,411,690],[367,12011,698],{"className":12012},[697],[367,12014,12016],{"className":12015},[670],[367,12017,12020],{"className":12018,"style":12019},[674],"height:0.5592em;",[367,12021],{},[367,12023,12024,12027],{"style":7900},[367,12025],{"className":12026,"style":1197},[682],[367,12028,12029],{},[367,12030,7704],{"className":12031},[1653,7909],[367,12033,698],{"className":12034},[697],[367,12036,12038],{"className":12037},[670],[367,12039,12042],{"className":12040,"style":12041},[674],"height:1.3989em;",[367,12043],{},[367,12045],{"className":12046,"style":1773},[711],[367,12048,12050,12053,12056,12059,12062,12065,12068,12071,12074,12077,12080,12083,12086,12118,12121,12154,12157],{"className":12049},[2170],[367,12051,2047],{"className":12052,"style":2178},[733,2177],[367,12054,5770],{"className":12055,"style":1021},[410,411],[367,12057,629],{"className":12058},[733],[367,12060,388],{"className":12061},[410,411],[367,12063,2656],{"className":12064},[2685],[367,12066],{"className":12067,"style":1773},[711],[367,12069,216],{"className":12070},[410,411],[367,12072,638],{"className":12073},[777],[367,12075],{"className":12076,"style":3132},[711],[367,12078,3029],{"className":12079},[3136],[367,12081],{"className":12082,"style":3132},[711],[367,12084,6657],{"className":12085,"style":6687},[410,411],[367,12087,12089,12092],{"className":12088},[410],[367,12090,8224],{"className":12091,"style":3132},[410,411],[367,12093,12095],{"className":12094},[661],[367,12096,12098],{"className":12097},[665],[367,12099,12101],{"className":12100},[670],[367,12102,12104],{"className":12103,"style":8447},[674],[367,12105,12106,12109],{"style":6904},[367,12107],{"className":12108,"style":683},[682],[367,12110,12112],{"className":12111},[687,688,689,690],[367,12113,12115],{"className":12114},[410,690],[367,12116,795],{"className":12117,"style":811},[410,411,690],[367,12119,629],{"className":12120},[733],[367,12122,12124,12127],{"className":12123},[410],[367,12125,388],{"className":12126},[410,411],[367,12128,12130],{"className":12129},[661],[367,12131,12133],{"className":12132},[665],[367,12134,12136],{"className":12135},[670],[367,12137,12140],{"className":12138,"style":12139},[674],"height:0.8019em;",[367,12141,12142,12145],{"style":6904},[367,12143],{"className":12144,"style":683},[682],[367,12146,12148],{"className":12147},[687,688,689,690],[367,12149,12151],{"className":12150},[410,690],[367,12152,11537],{"className":12153},[410,690],[367,12155,638],{"className":12156},[777],[367,12158,2062],{"className":12159,"style":2178},[777,2177],[367,12161,12162,12165],{"style":11791},[367,12163],{"className":12164,"style":11744},[682],[367,12166,12168,12171,12174,12177,12180,12269,12272],{"className":12167},[410],[367,12169],{"className":12170},[410],[367,12172],{"className":12173,"style":712},[711],[367,12175,623],{"className":12176},[716],[367,12178],{"className":12179,"style":712},[711],[367,12181,12183],{"className":12182},[410],[367,12184,12186],{"className":12185},[1653,3882],[367,12187,12189,12260],{"className":12188},[665,666],[367,12190,12192,12257],{"className":12191},[670],[367,12193,12195,12247],{"className":12194,"style":7876},[674],[367,12196,12198,12201],{"style":12197},"top:-2.3557em;margin-left:0em;",[367,12199],{"className":12200,"style":1197},[682],[367,12202,12204],{"className":12203},[687,688,689,690],[367,12205,12207,12241,12244],{"className":12206},[410,690],[367,12208,12210,12213],{"className":12209},[410,690],[367,12211,388],{"className":12212},[410,411,690],[367,12214,12216],{"className":12215},[661],[367,12217,12219],{"className":12218},[665],[367,12220,12222],{"className":12221},[670],[367,12223,12226],{"className":12224,"style":12225},[674],"height:0.6828em;",[367,12227,12229,12232],{"style":12228},"top:-2.786em;margin-right:0.0714em;",[367,12230],{"className":12231,"style":11992},[682],[367,12233,12235],{"className":12234},[687,11996,11997,690],[367,12236,12238],{"className":12237},[410,690],[367,12239,11537],{"className":12240},[410,690],[367,12242,835],{"className":12243},[716,690],[367,12245,1895],{"className":12246,"style":1922},[410,411,690],[367,12248,12249,12252],{"style":7900},[367,12250],{"className":12251,"style":1197},[682],[367,12253,12254],{},[367,12255,7704],{"className":12256},[1653,7909],[367,12258,698],{"className":12259},[697],[367,12261,12263],{"className":12262},[670],[367,12264,12267],{"className":12265,"style":12266},[674],"height:0.7443em;",[367,12268],{},[367,12270],{"className":12271,"style":1773},[711],[367,12273,12275,12282,12285,12288,12291,12294,12297,12300,12303,12306,12309,12312,12315,12402,12405,12522],{"className":12274},[2170],[367,12276,12278],{"className":12277,"style":2178},[733,2177],[367,12279,2047],{"className":12280},[3877,12281],"size2",[367,12283,5770],{"className":12284,"style":1021},[410,411],[367,12286,629],{"className":12287},[733],[367,12289,388],{"className":12290},[410,411],[367,12292,2656],{"className":12293},[2685],[367,12295],{"className":12296,"style":1773},[711],[367,12298,216],{"className":12299},[410,411],[367,12301,638],{"className":12302},[777],[367,12304],{"className":12305,"style":3132},[711],[367,12307,3029],{"className":12308},[3136],[367,12310],{"className":12311,"style":3132},[711],[367,12313,6657],{"className":12314,"style":6687},[410,411],[367,12316,12318],{"className":12317},[410],[367,12319,12321],{"className":12320},[1653,3882],[367,12322,12324,12393],{"className":12323},[665,666],[367,12325,12327,12390],{"className":12326},[670],[367,12328,12330,12380],{"className":12329,"style":7876},[674],[367,12331,12333,12336],{"style":12332},"top:-2.356em;margin-left:0em;",[367,12334],{"className":12335,"style":1197},[682],[367,12337,12339],{"className":12338},[687,688,689,690],[367,12340,12342,12374,12377],{"className":12341},[410,690],[367,12343,12345,12348],{"className":12344},[410,690],[367,12346,216],{"className":12347},[410,411,690],[367,12349,12351],{"className":12350},[661],[367,12352,12354],{"className":12353},[665],[367,12355,12357],{"className":12356},[670],[367,12358,12360],{"className":12359,"style":12225},[674],[367,12361,12362,12365],{"style":12228},[367,12363],{"className":12364,"style":11992},[682],[367,12366,12368],{"className":12367},[687,11996,11997,690],[367,12369,12371],{"className":12370},[410,690],[367,12372,11537],{"className":12373},[410,690],[367,12375,835],{"className":12376},[716,690],[367,12378,795],{"className":12379,"style":811},[410,411,690],[367,12381,12382,12385],{"style":7900},[367,12383],{"className":12384,"style":1197},[682],[367,12386,12387],{},[367,12388,7704],{"className":12389},[1653,7909],[367,12391,698],{"className":12392},[697],[367,12394,12396],{"className":12395},[670],[367,12397,12400],{"className":12398,"style":12399},[674],"height:0.744em;",[367,12401],{},[367,12403],{"className":12404,"style":1773},[711],[367,12406,12408,12411,12443,12446,12478,12481,12484,12516,12519],{"className":12407},[2170],[367,12409,2047],{"className":12410,"style":2178},[733,2177],[367,12412,12414,12417],{"className":12413},[410],[367,12415,8648],{"className":12416},[410,411],[367,12418,12420],{"className":12419},[661],[367,12421,12423],{"className":12422},[665],[367,12424,12426],{"className":12425},[670],[367,12427,12429],{"className":12428,"style":8447},[674],[367,12430,12431,12434],{"style":6904},[367,12432],{"className":12433,"style":683},[682],[367,12435,12437],{"className":12436},[687,688,689,690],[367,12438,12440],{"className":12439},[410,690],[367,12441,795],{"className":12442,"style":811},[410,411,690],[367,12444,629],{"className":12445},[733],[367,12447,12449,12452],{"className":12448},[410],[367,12450,388],{"className":12451},[410,411],[367,12453,12455],{"className":12454},[661],[367,12456,12458],{"className":12457},[665],[367,12459,12461],{"className":12460},[670],[367,12462,12464],{"className":12463,"style":12139},[674],[367,12465,12466,12469],{"style":6904},[367,12467],{"className":12468,"style":683},[682],[367,12470,12472],{"className":12471},[687,688,689,690],[367,12473,12475],{"className":12474},[410,690],[367,12476,11537],{"className":12477},[410,690],[367,12479,2656],{"className":12480},[2685],[367,12482],{"className":12483,"style":1773},[711],[367,12485,12487,12490],{"className":12486},[410],[367,12488,216],{"className":12489},[410,411],[367,12491,12493],{"className":12492},[661],[367,12494,12496],{"className":12495},[665],[367,12497,12499],{"className":12498},[670],[367,12500,12502],{"className":12501,"style":12139},[674],[367,12503,12504,12507],{"style":6904},[367,12505],{"className":12506,"style":683},[682],[367,12508,12510],{"className":12509},[687,688,689,690],[367,12511,12513],{"className":12512},[410,690],[367,12514,11537],{"className":12515},[410,690],[367,12517,638],{"className":12518},[777],[367,12520,2062],{"className":12521,"style":2178},[777,2177],[367,12523,12525],{"className":12524,"style":2178},[777,2177],[367,12526,2062],{"className":12527},[3877,12281],[367,12529,698],{"className":12530},[697],[367,12532,12534],{"className":12533},[670],[367,12535,12537],{"className":12536,"style":11857},[674],[367,12538],{},[212,12540,3218,12541,12626,12627,12749,12750,12812,12813,12626,12864,12940,12941,12626,13025,13168],{},[367,12542,12544,12566],{"className":12543},[370],[367,12545,12547],{"className":12546},[374],[376,12548,12549],{"xmlns":378},[380,12550,12551,12563],{},[383,12552,12553,12559,12561],{},[3653,12554,12555,12557],{},[386,12556,388],{},[621,12558,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,12560,835],{},[386,12562,1895],{},[390,12564,12565],{"encoding":392},"s' \\sim P",[367,12567,12569,12617],{"className":12568,"ariaHidden":397},[396],[367,12570,12572,12576,12608,12611,12614],{"className":12571},[401],[367,12573],{"className":12574,"style":12575},[405],"height:0.7519em;",[367,12577,12579,12582],{"className":12578},[410],[367,12580,388],{"className":12581},[410,411],[367,12583,12585],{"className":12584},[661],[367,12586,12588],{"className":12587},[665],[367,12589,12591],{"className":12590},[670],[367,12592,12594],{"className":12593,"style":12575},[674],[367,12595,12596,12599],{"style":4176},[367,12597],{"className":12598,"style":683},[682],[367,12600,12602],{"className":12601},[687,688,689,690],[367,12603,12605],{"className":12604},[410,690],[367,12606,11537],{"className":12607},[410,690],[367,12609],{"className":12610,"style":712},[711],[367,12612,835],{"className":12613},[716],[367,12615],{"className":12616,"style":712},[711],[367,12618,12620,12623],{"className":12619},[401],[367,12621],{"className":12622,"style":2333},[405],[367,12624,1895],{"className":12625,"style":1922},[410,411]," 是 ",[367,12628,12630,12666],{"className":12629},[370],[367,12631,12633],{"className":12632},[374],[376,12634,12635],{"xmlns":378},[380,12636,12637,12663],{},[383,12638,12639,12645,12647,12649,12651,12653,12655,12657,12659,12661],{},[3653,12640,12641,12643],{},[386,12642,388],{},[621,12644,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,12646,835],{},[386,12648,1895],{},[621,12650,629],{"stretchy":628},[621,12652,842],{},[386,12654,846],{"mathvariant":845},[386,12656,388],{},[621,12658,2656],{"separator":397},[386,12660,216],{},[621,12662,638],{"stretchy":628},[390,12664,12665],{"encoding":392},"s' \\sim P(\\cdot|s,a)",[367,12667,12669,12716],{"className":12668,"ariaHidden":397},[396],[367,12670,12672,12675,12707,12710,12713],{"className":12671},[401],[367,12673],{"className":12674,"style":12575},[405],[367,12676,12678,12681],{"className":12677},[410],[367,12679,388],{"className":12680},[410,411],[367,12682,12684],{"className":12683},[661],[367,12685,12687],{"className":12686},[665],[367,12688,12690],{"className":12689},[670],[367,12691,12693],{"className":12692,"style":12575},[674],[367,12694,12695,12698],{"style":4176},[367,12696],{"className":12697,"style":683},[682],[367,12699,12701],{"className":12700},[687,688,689,690],[367,12702,12704],{"className":12703},[410,690],[367,12705,11537],{"className":12706},[410,690],[367,12708],{"className":12709,"style":712},[711],[367,12711,835],{"className":12712},[716],[367,12714],{"className":12715,"style":712},[711],[367,12717,12719,12722,12725,12728,12731,12734,12737,12740,12743,12746],{"className":12718},[401],[367,12720],{"className":12721,"style":726},[405],[367,12723,1895],{"className":12724,"style":1922},[410,411],[367,12726,629],{"className":12727},[733],[367,12729,842],{"className":12730},[410],[367,12732,846],{"className":12733},[410],[367,12735,388],{"className":12736},[410,411],[367,12738,2656],{"className":12739},[2685],[367,12741],{"className":12742,"style":1773},[711],[367,12744,216],{"className":12745},[410,411],[367,12747,638],{"className":12748},[777]," 的简写，表示下一状态 ",[367,12751,12753,12771],{"className":12752},[370],[367,12754,12756],{"className":12755},[374],[376,12757,12758],{"xmlns":378},[380,12759,12760,12768],{},[383,12761,12762],{},[3653,12763,12764,12766],{},[386,12765,388],{},[621,12767,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[390,12769,12770],{"encoding":392},"s'",[367,12772,12774],{"className":12773,"ariaHidden":397},[396],[367,12775,12777,12780],{"className":12776},[401],[367,12778],{"className":12779,"style":12575},[405],[367,12781,12783,12786],{"className":12782},[410],[367,12784,388],{"className":12785},[410,411],[367,12787,12789],{"className":12788},[661],[367,12790,12792],{"className":12791},[665],[367,12793,12795],{"className":12794},[670],[367,12796,12798],{"className":12797,"style":12575},[674],[367,12799,12800,12803],{"style":4176},[367,12801],{"className":12802,"style":683},[682],[367,12804,12806],{"className":12805},[687,688,689,690],[367,12807,12809],{"className":12808},[410,690],[367,12810,11537],{"className":12811},[410,690]," 从环境的转移规则中采样；",[367,12814,12816,12834],{"className":12815},[370],[367,12817,12819],{"className":12818},[374],[376,12820,12821],{"xmlns":378},[380,12822,12823,12831],{},[383,12824,12825,12827,12829],{},[386,12826,216],{},[621,12828,835],{},[386,12830,795],{},[390,12832,12833],{"encoding":392},"a \\sim \\pi",[367,12835,12837,12855],{"className":12836,"ariaHidden":397},[396],[367,12838,12840,12843,12846,12849,12852],{"className":12839},[401],[367,12841],{"className":12842,"style":406},[405],[367,12844,216],{"className":12845},[410,411],[367,12847],{"className":12848,"style":712},[711],[367,12850,835],{"className":12851},[716],[367,12853],{"className":12854,"style":712},[711],[367,12856,12858,12861],{"className":12857},[401],[367,12859],{"className":12860,"style":406},[405],[367,12862,795],{"className":12863,"style":811},[410,411],[367,12865,12867,12895],{"className":12866},[370],[367,12868,12870],{"className":12869},[374],[376,12871,12872],{"xmlns":378},[380,12873,12874,12892],{},[383,12875,12876,12878,12880,12882,12884,12886,12888,12890],{},[386,12877,216],{},[621,12879,835],{},[386,12881,795],{},[621,12883,629],{"stretchy":628},[621,12885,842],{},[386,12887,846],{"mathvariant":845},[386,12889,388],{},[621,12891,638],{"stretchy":628},[390,12893,12894],{"encoding":392},"a \\sim \\pi(\\cdot|s)",[367,12896,12898,12916],{"className":12897,"ariaHidden":397},[396],[367,12899,12901,12904,12907,12910,12913],{"className":12900},[401],[367,12902],{"className":12903,"style":406},[405],[367,12905,216],{"className":12906},[410,411],[367,12908],{"className":12909,"style":712},[711],[367,12911,835],{"className":12912},[716],[367,12914],{"className":12915,"style":712},[711],[367,12917,12919,12922,12925,12928,12931,12934,12937],{"className":12918},[401],[367,12920],{"className":12921,"style":726},[405],[367,12923,795],{"className":12924,"style":811},[410,411],[367,12926,629],{"className":12927},[733],[367,12929,842],{"className":12930},[410],[367,12932,846],{"className":12933},[410],[367,12935,388],{"className":12936},[410,411],[367,12938,638],{"className":12939},[777]," 的简写；",[367,12942,12944,12966],{"className":12943},[370],[367,12945,12947],{"className":12946},[374],[376,12948,12949],{"xmlns":378},[380,12950,12951,12963],{},[383,12952,12953,12959,12961],{},[3653,12954,12955,12957],{},[386,12956,216],{},[621,12958,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,12960,835],{},[386,12962,795],{},[390,12964,12965],{"encoding":392},"a' \\sim \\pi",[367,12967,12969,13016],{"className":12968,"ariaHidden":397},[396],[367,12970,12972,12975,13007,13010,13013],{"className":12971},[401],[367,12973],{"className":12974,"style":12575},[405],[367,12976,12978,12981],{"className":12977},[410],[367,12979,216],{"className":12980},[410,411],[367,12982,12984],{"className":12983},[661],[367,12985,12987],{"className":12986},[665],[367,12988,12990],{"className":12989},[670],[367,12991,12993],{"className":12992,"style":12575},[674],[367,12994,12995,12998],{"style":4176},[367,12996],{"className":12997,"style":683},[682],[367,12999,13001],{"className":13000},[687,688,689,690],[367,13002,13004],{"className":13003},[410,690],[367,13005,11537],{"className":13006},[410,690],[367,13008],{"className":13009,"style":712},[711],[367,13011,835],{"className":13012},[716],[367,13014],{"className":13015,"style":712},[711],[367,13017,13019,13022],{"className":13018},[401],[367,13020],{"className":13021,"style":406},[405],[367,13023,795],{"className":13024,"style":811},[410,411],[367,13026,13028,13064],{"className":13027},[370],[367,13029,13031],{"className":13030},[374],[376,13032,13033],{"xmlns":378},[380,13034,13035,13061],{},[383,13036,13037,13043,13045,13047,13049,13051,13053,13059],{},[3653,13038,13039,13041],{},[386,13040,216],{},[621,13042,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,13044,835],{},[386,13046,795],{},[621,13048,629],{"stretchy":628},[621,13050,842],{},[386,13052,846],{"mathvariant":845},[3653,13054,13055,13057],{},[386,13056,388],{},[621,13058,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,13060,638],{"stretchy":628},[390,13062,13063],{"encoding":392},"a' \\sim \\pi(\\cdot|s')",[367,13065,13067,13114],{"className":13066,"ariaHidden":397},[396],[367,13068,13070,13073,13105,13108,13111],{"className":13069},[401],[367,13071],{"className":13072,"style":12575},[405],[367,13074,13076,13079],{"className":13075},[410],[367,13077,216],{"className":13078},[410,411],[367,13080,13082],{"className":13081},[661],[367,13083,13085],{"className":13084},[665],[367,13086,13088],{"className":13087},[670],[367,13089,13091],{"className":13090,"style":12575},[674],[367,13092,13093,13096],{"style":4176},[367,13094],{"className":13095,"style":683},[682],[367,13097,13099],{"className":13098},[687,688,689,690],[367,13100,13102],{"className":13101},[410,690],[367,13103,11537],{"className":13104},[410,690],[367,13106],{"className":13107,"style":712},[711],[367,13109,835],{"className":13110},[716],[367,13112],{"className":13113,"style":712},[711],[367,13115,13117,13121,13124,13127,13130,13133,13165],{"className":13116},[401],[367,13118],{"className":13119,"style":13120},[405],"height:1.0019em;vertical-align:-0.25em;",[367,13122,795],{"className":13123,"style":811},[410,411],[367,13125,629],{"className":13126},[733],[367,13128,842],{"className":13129},[410],[367,13131,846],{"className":13132},[410],[367,13134,13136,13139],{"className":13135},[410],[367,13137,388],{"className":13138},[410,411],[367,13140,13142],{"className":13141},[661],[367,13143,13145],{"className":13144},[665],[367,13146,13148],{"className":13147},[670],[367,13149,13151],{"className":13150,"style":12575},[674],[367,13152,13153,13156],{"style":4176},[367,13154],{"className":13155,"style":683},[682],[367,13157,13159],{"className":13158},[687,688,689,690],[367,13160,13162],{"className":13161},[410,690],[367,13163,11537],{"className":13164},[410,690],[367,13166,638],{"className":13167},[777]," 的简写。",[212,13170,13171],{},"最优价值函数的贝尔曼方程为：",[367,13173,13175],{"className":13174},[598],[367,13176,13178,13402],{"className":13177},[370],[367,13179,13181],{"className":13180},[374],[376,13182,13183],{"xmlns":378,"display":607},[380,13184,13185,13399],{},[1069,13186,13187,13285],{"rowspacing":1071,"columnalign":1072,"columnspacing":1073},[1075,13188,13189,13207],{},[1078,13190,13191],{},[1081,13192,13193],{"scriptlevel":1083,"displaystyle":397},[383,13194,13195,13201,13203,13205],{},[3653,13196,13197,13199],{},[386,13198,8224],{},[621,13200,7962],{},[621,13202,629],{"stretchy":628},[386,13204,388],{},[621,13206,638],{"stretchy":628},[1078,13208,13209],{},[1081,13210,13211],{"scriptlevel":1083,"displaystyle":397},[383,13212,13213,13215,13217,13227,13247],{},[383,13214],{},[621,13216,623],{},[7697,13218,13219,13225],{},[383,13220,13221,13223],{},[386,13222,7976],{},[621,13224,1637],{},[386,13226,216],{},[386,13228,13229],{},[7697,13230,13231,13235],{},[621,13232,13233],{},[386,13234,7704],{"mathvariant":7703},[383,13236,13237,13243,13245],{},[3653,13238,13239,13241],{},[386,13240,388],{},[621,13242,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,13244,835],{},[386,13246,1895],{},[383,13248,13249,13251,13253,13255,13257,13259,13261,13263,13265,13267,13273,13275,13281,13283],{},[621,13250,2047],{"fence":397},[386,13252,5770],{},[621,13254,629],{"stretchy":628},[386,13256,388],{},[621,13258,2656],{"separator":397},[386,13260,216],{},[621,13262,638],{"stretchy":628},[621,13264,3029],{},[386,13266,6657],{},[3653,13268,13269,13271],{},[386,13270,8224],{},[621,13272,7962],{},[621,13274,629],{"stretchy":628},[3653,13276,13277,13279],{},[386,13278,388],{},[621,13280,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,13282,638],{"stretchy":628},[621,13284,2062],{"fence":397},[1075,13286,13287,13309],{},[1078,13288,13289],{},[1081,13290,13291],{"scriptlevel":1083,"displaystyle":397},[383,13292,13293,13299,13301,13303,13305,13307],{},[3653,13294,13295,13297],{},[386,13296,8648],{},[621,13298,7962],{},[621,13300,629],{"stretchy":628},[386,13302,388],{},[621,13304,2656],{"separator":397},[386,13306,216],{},[621,13308,638],{"stretchy":628},[1078,13310,13311],{},[1081,13312,13313],{"scriptlevel":1083,"displaystyle":397},[383,13314,13315,13317,13319,13339],{},[383,13316],{},[621,13318,623],{},[386,13320,13321],{},[7697,13322,13323,13327],{},[621,13324,13325],{},[386,13326,7704],{"mathvariant":7703},[383,13328,13329,13335,13337],{},[3653,13330,13331,13333],{},[386,13332,388],{},[621,13334,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,13336,835],{},[386,13338,1895],{},[383,13340,13341,13343,13345,13347,13349,13351,13353,13355,13357,13359,13373,13379,13381,13387,13389,13395,13397],{},[621,13342,2047],{"fence":397},[386,13344,5770],{},[621,13346,629],{"stretchy":628},[386,13348,388],{},[621,13350,2656],{"separator":397},[386,13352,216],{},[621,13354,638],{"stretchy":628},[621,13356,3029],{},[386,13358,6657],{},[7697,13360,13361,13367],{},[383,13362,13363,13365],{},[386,13364,7976],{},[621,13366,1637],{},[3653,13368,13369,13371],{},[386,13370,216],{},[621,13372,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[3653,13374,13375,13377],{},[386,13376,8648],{},[621,13378,7962],{},[621,13380,629],{"stretchy":628},[3653,13382,13383,13385],{},[386,13384,388],{},[621,13386,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,13388,2656],{"separator":397},[3653,13390,13391,13393],{},[386,13392,216],{},[621,13394,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[621,13396,638],{"stretchy":628},[621,13398,2062],{"fence":397},[390,13400,13401],{"encoding":392},"\\begin{aligned}\nV^*(s) &= \\max_a \\underset{s'\\sim P}{\\mathbb{E}}\\left[r(s,a) + \\gamma V^*(s')\\right] \\\\\nQ^*(s,a) &= \\underset{s'\\sim P}{\\mathbb{E}}\\left[r(s,a) + \\gamma \\max_{a'} Q^*(s',a')\\right]\n\\end{aligned}",[367,13403,13405],{"className":13404,"ariaHidden":397},[396],[367,13406,13408,13412],{"className":13407},[401],[367,13409],{"className":13410,"style":13411},[405],"height:3.7787em;vertical-align:-1.6393em;",[367,13413,13415],{"className":13414},[410],[367,13416,13418,13546],{"className":13417},[1069],[367,13419,13421],{"className":13420},[1180],[367,13422,13424,13537],{"className":13423},[665,666],[367,13425,13427,13534],{"className":13426},[670],[367,13428,13431,13478],{"className":13429,"style":13430},[674],"height:2.1393em;",[367,13432,13434,13437],{"style":13433},"top:-4.4493em;",[367,13435],{"className":13436,"style":11744},[682],[367,13438,13440,13469,13472,13475],{"className":13439},[410],[367,13441,13443,13446],{"className":13442},[410],[367,13444,8224],{"className":13445,"style":3132},[410,411],[367,13447,13449],{"className":13448},[661],[367,13450,13452],{"className":13451},[665],[367,13453,13455],{"className":13454},[670],[367,13456,13458],{"className":13457,"style":8001},[674],[367,13459,13460,13463],{"style":6904},[367,13461],{"className":13462,"style":683},[682],[367,13464,13466],{"className":13465},[687,688,689,690],[367,13467,7962],{"className":13468},[3136,690],[367,13470,629],{"className":13471},[733],[367,13473,388],{"className":13474},[410,411],[367,13476,638],{"className":13477},[777],[367,13479,13481,13484],{"style":13480},"top:-2.255em;",[367,13482],{"className":13483,"style":11744},[682],[367,13485,13487,13516,13519,13522,13525,13528,13531],{"className":13486},[410],[367,13488,13490,13493],{"className":13489},[410],[367,13491,8648],{"className":13492},[410,411],[367,13494,13496],{"className":13495},[661],[367,13497,13499],{"className":13498},[665],[367,13500,13502],{"className":13501},[670],[367,13503,13505],{"className":13504,"style":8001},[674],[367,13506,13507,13510],{"style":6904},[367,13508],{"className":13509,"style":683},[682],[367,13511,13513],{"className":13512},[687,688,689,690],[367,13514,7962],{"className":13515},[3136,690],[367,13517,629],{"className":13518},[733],[367,13520,388],{"className":13521},[410,411],[367,13523,2656],{"className":13524},[2685],[367,13526],{"className":13527,"style":1773},[711],[367,13529,216],{"className":13530},[410,411],[367,13532,638],{"className":13533},[777],[367,13535,698],{"className":13536},[697],[367,13538,13540],{"className":13539},[670],[367,13541,13544],{"className":13542,"style":13543},[674],"height:1.6393em;",[367,13545],{},[367,13547,13549],{"className":13548},[1305],[367,13550,13552,14168],{"className":13551},[665,666],[367,13553,13555,14165],{"className":13554},[670],[367,13556,13558,13822],{"className":13557,"style":13430},[674],[367,13559,13560,13563],{"style":13433},[367,13561],{"className":13562,"style":11744},[682],[367,13564,13566,13569,13572,13575,13578,13622,13625,13710,13713],{"className":13565},[410],[367,13567],{"className":13568},[410],[367,13570],{"className":13571,"style":712},[711],[367,13573,623],{"className":13574},[716],[367,13576],{"className":13577,"style":712},[711],[367,13579,13581],{"className":13580},[1653,3882],[367,13582,13584,13614],{"className":13583},[665,666],[367,13585,13587,13611],{"className":13586},[670],[367,13588,13590,13601],{"className":13589,"style":406},[674],[367,13591,13592,13595],{"style":7879},[367,13593],{"className":13594,"style":1197},[682],[367,13596,13598],{"className":13597},[687,688,689,690],[367,13599,216],{"className":13600},[410,411,690],[367,13602,13603,13606],{"style":7900},[367,13604],{"className":13605,"style":1197},[682],[367,13607,13608],{},[367,13609,7976],{"className":13610},[1653],[367,13612,698],{"className":13613},[697],[367,13615,13617],{"className":13616},[670],[367,13618,13620],{"className":13619,"style":7919},[674],[367,13621],{},[367,13623],{"className":13624,"style":1773},[711],[367,13626,13628],{"className":13627},[410],[367,13629,13631],{"className":13630},[1653,3882],[367,13632,13634,13702],{"className":13633},[665,666],[367,13635,13637,13699],{"className":13636},[670],[367,13638,13640,13689],{"className":13639,"style":7876},[674],[367,13641,13642,13645],{"style":12197},[367,13643],{"className":13644,"style":1197},[682],[367,13646,13648],{"className":13647},[687,688,689,690],[367,13649,13651,13683,13686],{"className":13650},[410,690],[367,13652,13654,13657],{"className":13653},[410,690],[367,13655,388],{"className":13656},[410,411,690],[367,13658,13660],{"className":13659},[661],[367,13661,13663],{"className":13662},[665],[367,13664,13666],{"className":13665},[670],[367,13667,13669],{"className":13668,"style":12225},[674],[367,13670,13671,13674],{"style":12228},[367,13672],{"className":13673,"style":11992},[682],[367,13675,13677],{"className":13676},[687,11996,11997,690],[367,13678,13680],{"className":13679},[410,690],[367,13681,11537],{"className":13682},[410,690],[367,13684,835],{"className":13685},[716,690],[367,13687,1895],{"className":13688,"style":1922},[410,411,690],[367,13690,13691,13694],{"style":7900},[367,13692],{"className":13693,"style":1197},[682],[367,13695,13696],{},[367,13697,7704],{"className":13698},[1653,7909],[367,13700,698],{"className":13701},[697],[367,13703,13705],{"className":13704},[670],[367,13706,13708],{"className":13707,"style":12266},[674],[367,13709],{},[367,13711],{"className":13712,"style":1773},[711],[367,13714,13716,13719,13722,13725,13728,13731,13734,13737,13740,13743,13746,13749,13752,13781,13784,13816,13819],{"className":13715},[2170],[367,13717,2047],{"className":13718,"style":2178},[733,2177],[367,13720,5770],{"className":13721,"style":1021},[410,411],[367,13723,629],{"className":13724},[733],[367,13726,388],{"className":13727},[410,411],[367,13729,2656],{"className":13730},[2685],[367,13732],{"className":13733,"style":1773},[711],[367,13735,216],{"className":13736},[410,411],[367,13738,638],{"className":13739},[777],[367,13741],{"className":13742,"style":3132},[711],[367,13744,3029],{"className":13745},[3136],[367,13747],{"className":13748,"style":3132},[711],[367,13750,6657],{"className":13751,"style":6687},[410,411],[367,13753,13755,13758],{"className":13754},[410],[367,13756,8224],{"className":13757,"style":3132},[410,411],[367,13759,13761],{"className":13760},[661],[367,13762,13764],{"className":13763},[665],[367,13765,13767],{"className":13766},[670],[367,13768,13770],{"className":13769,"style":8001},[674],[367,13771,13772,13775],{"style":6904},[367,13773],{"className":13774,"style":683},[682],[367,13776,13778],{"className":13777},[687,688,689,690],[367,13779,7962],{"className":13780},[3136,690],[367,13782,629],{"className":13783},[733],[367,13785,13787,13790],{"className":13786},[410],[367,13788,388],{"className":13789},[410,411],[367,13791,13793],{"className":13792},[661],[367,13794,13796],{"className":13795},[665],[367,13797,13799],{"className":13798},[670],[367,13800,13802],{"className":13801,"style":12139},[674],[367,13803,13804,13807],{"style":6904},[367,13805],{"className":13806,"style":683},[682],[367,13808,13810],{"className":13809},[687,688,689,690],[367,13811,13813],{"className":13812},[410,690],[367,13814,11537],{"className":13815},[410,690],[367,13817,638],{"className":13818},[777],[367,13820,2062],{"className":13821,"style":2178},[777,2177],[367,13823,13824,13827],{"style":13480},[367,13825],{"className":13826,"style":11744},[682],[367,13828,13830,13833,13836,13839,13842,13927,13930],{"className":13829},[410],[367,13831],{"className":13832},[410],[367,13834],{"className":13835,"style":712},[711],[367,13837,623],{"className":13838},[716],[367,13840],{"className":13841,"style":712},[711],[367,13843,13845],{"className":13844},[410],[367,13846,13848],{"className":13847},[1653,3882],[367,13849,13851,13919],{"className":13850},[665,666],[367,13852,13854,13916],{"className":13853},[670],[367,13855,13857,13906],{"className":13856,"style":7876},[674],[367,13858,13859,13862],{"style":12197},[367,13860],{"className":13861,"style":1197},[682],[367,13863,13865],{"className":13864},[687,688,689,690],[367,13866,13868,13900,13903],{"className":13867},[410,690],[367,13869,13871,13874],{"className":13870},[410,690],[367,13872,388],{"className":13873},[410,411,690],[367,13875,13877],{"className":13876},[661],[367,13878,13880],{"className":13879},[665],[367,13881,13883],{"className":13882},[670],[367,13884,13886],{"className":13885,"style":12225},[674],[367,13887,13888,13891],{"style":12228},[367,13889],{"className":13890,"style":11992},[682],[367,13892,13894],{"className":13893},[687,11996,11997,690],[367,13895,13897],{"className":13896},[410,690],[367,13898,11537],{"className":13899},[410,690],[367,13901,835],{"className":13902},[716,690],[367,13904,1895],{"className":13905,"style":1922},[410,411,690],[367,13907,13908,13911],{"style":7900},[367,13909],{"className":13910,"style":1197},[682],[367,13912,13913],{},[367,13914,7704],{"className":13915},[1653,7909],[367,13917,698],{"className":13918},[697],[367,13920,13922],{"className":13921},[670],[367,13923,13925],{"className":13924,"style":12266},[674],[367,13926],{},[367,13928],{"className":13929,"style":1773},[711],[367,13931,13933,13939,13942,13945,13948,13951,13954,13957,13960,13963,13966,13969,13972,13975,14051,14054,14083,14086,14118,14121,14124,14156,14159],{"className":13932},[2170],[367,13934,13936],{"className":13935,"style":2178},[733,2177],[367,13937,2047],{"className":13938},[3877,12281],[367,13940,5770],{"className":13941,"style":1021},[410,411],[367,13943,629],{"className":13944},[733],[367,13946,388],{"className":13947},[410,411],[367,13949,2656],{"className":13950},[2685],[367,13952],{"className":13953,"style":1773},[711],[367,13955,216],{"className":13956},[410,411],[367,13958,638],{"className":13959},[777],[367,13961],{"className":13962,"style":3132},[711],[367,13964,3029],{"className":13965},[3136],[367,13967],{"className":13968,"style":3132},[711],[367,13970,6657],{"className":13971,"style":6687},[410,411],[367,13973],{"className":13974,"style":1773},[711],[367,13976,13978],{"className":13977},[1653,3882],[367,13979,13981,14043],{"className":13980},[665,666],[367,13982,13984,14040],{"className":13983},[670],[367,13985,13987,14030],{"className":13986,"style":406},[674],[367,13988,13989,13992],{"style":12332},[367,13990],{"className":13991,"style":1197},[682],[367,13993,13995],{"className":13994},[687,688,689,690],[367,13996,13998],{"className":13997},[410,690],[367,13999,14001,14004],{"className":14000},[410,690],[367,14002,216],{"className":14003},[410,411,690],[367,14005,14007],{"className":14006},[661],[367,14008,14010],{"className":14009},[665],[367,14011,14013],{"className":14012},[670],[367,14014,14016],{"className":14015,"style":12225},[674],[367,14017,14018,14021],{"style":12228},[367,14019],{"className":14020,"style":11992},[682],[367,14022,14024],{"className":14023},[687,11996,11997,690],[367,14025,14027],{"className":14026},[410,690],[367,14028,11537],{"className":14029},[410,690],[367,14031,14032,14035],{"style":7900},[367,14033],{"className":14034,"style":1197},[682],[367,14036,14037],{},[367,14038,7976],{"className":14039},[1653],[367,14041,698],{"className":14042},[697],[367,14044,14046],{"className":14045},[670],[367,14047,14049],{"className":14048,"style":12399},[674],[367,14050],{},[367,14052],{"className":14053,"style":1773},[711],[367,14055,14057,14060],{"className":14056},[410],[367,14058,8648],{"className":14059},[410,411],[367,14061,14063],{"className":14062},[661],[367,14064,14066],{"className":14065},[665],[367,14067,14069],{"className":14068},[670],[367,14070,14072],{"className":14071,"style":8001},[674],[367,14073,14074,14077],{"style":6904},[367,14075],{"className":14076,"style":683},[682],[367,14078,14080],{"className":14079},[687,688,689,690],[367,14081,7962],{"className":14082},[3136,690],[367,14084,629],{"className":14085},[733],[367,14087,14089,14092],{"className":14088},[410],[367,14090,388],{"className":14091},[410,411],[367,14093,14095],{"className":14094},[661],[367,14096,14098],{"className":14097},[665],[367,14099,14101],{"className":14100},[670],[367,14102,14104],{"className":14103,"style":12139},[674],[367,14105,14106,14109],{"style":6904},[367,14107],{"className":14108,"style":683},[682],[367,14110,14112],{"className":14111},[687,688,689,690],[367,14113,14115],{"className":14114},[410,690],[367,14116,11537],{"className":14117},[410,690],[367,14119,2656],{"className":14120},[2685],[367,14122],{"className":14123,"style":1773},[711],[367,14125,14127,14130],{"className":14126},[410],[367,14128,216],{"className":14129},[410,411],[367,14131,14133],{"className":14132},[661],[367,14134,14136],{"className":14135},[665],[367,14137,14139],{"className":14138},[670],[367,14140,14142],{"className":14141,"style":12139},[674],[367,14143,14144,14147],{"style":6904},[367,14145],{"className":14146,"style":683},[682],[367,14148,14150],{"className":14149},[687,688,689,690],[367,14151,14153],{"className":14152},[410,690],[367,14154,11537],{"className":14155},[410,690],[367,14157,638],{"className":14158},[777],[367,14160,14162],{"className":14161,"style":2178},[777,2177],[367,14163,2062],{"className":14164},[3877,12281],[367,14166,698],{"className":14167},[697],[367,14169,14171],{"className":14170},[670],[367,14172,14174],{"className":14173,"style":13543},[674],[367,14175],{},[212,14177,14178,14179,272,14210,14240],{},"同策略价值函数与最优价值函数的贝尔曼方程之间，关键区别在对动作取不取 ",[367,14180,14182,14198],{"className":14181},[370],[367,14183,14185],{"className":14184},[374],[376,14186,14187],{"xmlns":378},[380,14188,14189,14195],{},[383,14190,14191,14193],{},[386,14192,7976],{},[621,14194,1637],{},[390,14196,14197],{"encoding":392},"\\max",[367,14199,14201],{"className":14200,"ariaHidden":397},[396],[367,14202,14204,14207],{"className":14203},[401],[367,14205],{"className":14206,"style":406},[405],[367,14208,7976],{"className":14209},[1653],[367,14211,14213,14228],{"className":14212},[370],[367,14214,14216],{"className":14215},[374],[376,14217,14218],{"xmlns":378},[380,14219,14220,14226],{},[383,14221,14222,14224],{},[386,14223,7976],{},[621,14225,1637],{},[390,14227,14197],{"encoding":392},[367,14229,14231],{"className":14230,"ariaHidden":397},[396],[367,14232,14234,14237],{"className":14233},[401],[367,14235],{"className":14236,"style":406},[405],[367,14238,7976],{"className":14239},[1653]," 的存在反映了这样一个事实：每当智能体有机会自己挑动作，要想最优地行动，就必须挑那个带来最高价值的动作。",[208,14242,14243],{"title":462},[212,14244,14245],{},"「贝尔曼备份（Bellman backup）」这个词在 RL 文献中出现得非常频繁。一个状态或状态-动作对的贝尔曼备份，就是贝尔曼方程的右边：奖励加下一状态的价值。",[357,14247,14248],{"id":14248},"优势函数",[212,14250,14251,14252,14255,14256,14259],{},"有时在 RL 中，我们不需要绝对地描述一个动作有多好，只需要知道它平均而言比别的动作好多少。也就是说，我们想知道该动作的相对",[315,14253,14254],{},"优势（advantage）","。我们用",[315,14257,14258],{},"优势函数（advantage function）"," 把这个概念精确化。",[212,14261,14262,14263,14291,14292,14383,14384,14412,14413,14441,14442,14496,14497,14525],{},"对应策略 ",[367,14264,14266,14279],{"className":14265},[370],[367,14267,14269],{"className":14268},[374],[376,14270,14271],{"xmlns":378},[380,14272,14273,14277],{},[383,14274,14275],{},[386,14276,795],{},[390,14278,798],{"encoding":392},[367,14280,14282],{"className":14281,"ariaHidden":397},[396],[367,14283,14285,14288],{"className":14284},[401],[367,14286],{"className":14287,"style":406},[405],[367,14289,795],{"className":14290,"style":811},[410,411]," 的优势函数 ",[367,14293,14295,14324],{"className":14294},[370],[367,14296,14298],{"className":14297},[374],[376,14299,14300],{"xmlns":378},[380,14301,14302,14321],{},[383,14303,14304,14311,14313,14315,14317,14319],{},[3653,14305,14306,14309],{},[386,14307,14308],{},"A",[386,14310,795],{},[621,14312,629],{"stretchy":628},[386,14314,388],{},[621,14316,2656],{"separator":397},[386,14318,216],{},[621,14320,638],{"stretchy":628},[390,14322,14323],{"encoding":392},"A^{\\pi}(s,a)",[367,14325,14327],{"className":14326,"ariaHidden":397},[396],[367,14328,14330,14333,14365,14368,14371,14374,14377,14380],{"className":14329},[401],[367,14331],{"className":14332,"style":726},[405],[367,14334,14336,14339],{"className":14335},[410],[367,14337,14308],{"className":14338},[410,411],[367,14340,14342],{"className":14341},[661],[367,14343,14345],{"className":14344},[665],[367,14346,14348],{"className":14347},[670],[367,14349,14351],{"className":14350,"style":8263},[674],[367,14352,14353,14356],{"style":4176},[367,14354],{"className":14355,"style":683},[682],[367,14357,14359],{"className":14358},[687,688,689,690],[367,14360,14362],{"className":14361},[410,690],[367,14363,795],{"className":14364,"style":811},[410,411,690],[367,14366,629],{"className":14367},[733],[367,14369,388],{"className":14370},[410,411],[367,14372,2656],{"className":14373},[2685],[367,14375],{"className":14376,"style":1773},[711],[367,14378,216],{"className":14379},[410,411],[367,14381,638],{"className":14382},[777]," 描述的是：在状态 ",[367,14385,14387,14400],{"className":14386},[370],[367,14388,14390],{"className":14389},[374],[376,14391,14392],{"xmlns":378},[380,14393,14394,14398],{},[383,14395,14396],{},[386,14397,388],{},[390,14399,388],{"encoding":392},[367,14401,14403],{"className":14402,"ariaHidden":397},[396],[367,14404,14406,14409],{"className":14405},[401],[367,14407],{"className":14408,"style":406},[405],[367,14410,388],{"className":14411},[410,411]," 采取特定动作 ",[367,14414,14416,14429],{"className":14415},[370],[367,14417,14419],{"className":14418},[374],[376,14420,14421],{"xmlns":378},[380,14422,14423,14427],{},[383,14424,14425],{},[386,14426,216],{},[390,14428,216],{"encoding":392},[367,14430,14432],{"className":14431,"ariaHidden":397},[396],[367,14433,14435,14438],{"className":14434},[401],[367,14436],{"className":14437,"style":406},[405],[367,14439,216],{"className":14440},[410,411],"，比按 ",[367,14443,14445,14469],{"className":14444},[370],[367,14446,14448],{"className":14447},[374],[376,14449,14450],{"xmlns":378},[380,14451,14452,14466],{},[383,14453,14454,14456,14458,14460,14462,14464],{},[386,14455,795],{},[621,14457,629],{"stretchy":628},[621,14459,842],{},[386,14461,846],{"mathvariant":845},[386,14463,388],{},[621,14465,638],{"stretchy":628},[390,14467,14468],{"encoding":392},"\\pi(\\cdot|s)",[367,14470,14472],{"className":14471,"ariaHidden":397},[396],[367,14473,14475,14478,14481,14484,14487,14490,14493],{"className":14474},[401],[367,14476],{"className":14477,"style":726},[405],[367,14479,795],{"className":14480,"style":811},[410,411],[367,14482,629],{"className":14483},[733],[367,14485,842],{"className":14486},[410],[367,14488,846],{"className":14489},[410],[367,14491,388],{"className":14492},[410,411],[367,14494,638],{"className":14495},[777]," 随机选一个动作好多少；这里假设之后一直按 ",[367,14498,14500,14513],{"className":14499},[370],[367,14501,14503],{"className":14502},[374],[376,14504,14505],{"xmlns":378},[380,14506,14507,14511],{},[383,14508,14509],{},[386,14510,795],{},[390,14512,798],{"encoding":392},[367,14514,14516],{"className":14515,"ariaHidden":397},[396],[367,14517,14519,14522],{"className":14518},[401],[367,14520],{"className":14521,"style":406},[405],[367,14523,795],{"className":14524,"style":811},[410,411]," 行动。数学上，优势函数定义为：",[367,14527,14529],{"className":14528},[598],[367,14530,14532,14592],{"className":14531},[370],[367,14533,14535],{"className":14534},[374],[376,14536,14537],{"xmlns":378,"display":607},[380,14538,14539,14589],{},[383,14540,14541,14547,14549,14551,14553,14555,14557,14559,14565,14567,14569,14571,14573,14575,14577,14583,14585,14587],{},[3653,14542,14543,14545],{},[386,14544,14308],{},[386,14546,795],{},[621,14548,629],{"stretchy":628},[386,14550,388],{},[621,14552,2656],{"separator":397},[386,14554,216],{},[621,14556,638],{"stretchy":628},[621,14558,623],{},[3653,14560,14561,14563],{},[386,14562,8648],{},[386,14564,795],{},[621,14566,629],{"stretchy":628},[386,14568,388],{},[621,14570,2656],{"separator":397},[386,14572,216],{},[621,14574,638],{"stretchy":628},[621,14576,2650],{},[3653,14578,14579,14581],{},[386,14580,8224],{},[386,14582,795],{},[621,14584,629],{"stretchy":628},[386,14586,388],{},[621,14588,638],{"stretchy":628},[390,14590,14591],{"encoding":392},"A^{\\pi}(s,a) = Q^{\\pi}(s,a) - V^{\\pi}(s)",[367,14593,14595,14660,14725],{"className":14594,"ariaHidden":397},[396],[367,14596,14598,14601,14633,14636,14639,14642,14645,14648,14651,14654,14657],{"className":14597},[401],[367,14599],{"className":14600,"style":726},[405],[367,14602,14604,14607],{"className":14603},[410],[367,14605,14308],{"className":14606},[410,411],[367,14608,14610],{"className":14609},[661],[367,14611,14613],{"className":14612},[665],[367,14614,14616],{"className":14615},[670],[367,14617,14619],{"className":14618,"style":8447},[674],[367,14620,14621,14624],{"style":6904},[367,14622],{"className":14623,"style":683},[682],[367,14625,14627],{"className":14626},[687,688,689,690],[367,14628,14630],{"className":14629},[410,690],[367,14631,795],{"className":14632,"style":811},[410,411,690],[367,14634,629],{"className":14635},[733],[367,14637,388],{"className":14638},[410,411],[367,14640,2656],{"className":14641},[2685],[367,14643],{"className":14644,"style":1773},[711],[367,14646,216],{"className":14647},[410,411],[367,14649,638],{"className":14650},[777],[367,14652],{"className":14653,"style":712},[711],[367,14655,623],{"className":14656},[716],[367,14658],{"className":14659,"style":712},[711],[367,14661,14663,14666,14698,14701,14704,14707,14710,14713,14716,14719,14722],{"className":14662},[401],[367,14664],{"className":14665,"style":726},[405],[367,14667,14669,14672],{"className":14668},[410],[367,14670,8648],{"className":14671},[410,411],[367,14673,14675],{"className":14674},[661],[367,14676,14678],{"className":14677},[665],[367,14679,14681],{"className":14680},[670],[367,14682,14684],{"className":14683,"style":8447},[674],[367,14685,14686,14689],{"style":6904},[367,14687],{"className":14688,"style":683},[682],[367,14690,14692],{"className":14691},[687,688,689,690],[367,14693,14695],{"className":14694},[410,690],[367,14696,795],{"className":14697,"style":811},[410,411,690],[367,14699,629],{"className":14700},[733],[367,14702,388],{"className":14703},[410,411],[367,14705,2656],{"className":14706},[2685],[367,14708],{"className":14709,"style":1773},[711],[367,14711,216],{"className":14712},[410,411],[367,14714,638],{"className":14715},[777],[367,14717],{"className":14718,"style":3132},[711],[367,14720,2650],{"className":14721},[3136],[367,14723],{"className":14724,"style":3132},[711],[367,14726,14728,14731,14763,14766,14769],{"className":14727},[401],[367,14729],{"className":14730,"style":726},[405],[367,14732,14734,14737],{"className":14733},[410],[367,14735,8224],{"className":14736,"style":3132},[410,411],[367,14738,14740],{"className":14739},[661],[367,14741,14743],{"className":14742},[665],[367,14744,14746],{"className":14745},[670],[367,14747,14749],{"className":14748,"style":8447},[674],[367,14750,14751,14754],{"style":6904},[367,14752],{"className":14753,"style":683},[682],[367,14755,14757],{"className":14756},[687,688,689,690],[367,14758,14760],{"className":14759},[410,690],[367,14761,795],{"className":14762,"style":811},[410,411,690],[367,14764,629],{"className":14765},[733],[367,14767,388],{"className":14768},[410,411],[367,14770,638],{"className":14771},[777],[208,14773,14774],{"title":462},[212,14775,14776],{},"这一点后面还会细讲：优势函数对策略梯度方法至关重要。",[254,14778,14780],{"id":14779},"选读形式化","（选读）形式化",[212,14782,14783,14784,14787,14788,14924],{},"到目前为止，我们一直在非正式地讨论智能体所处的环境。但如果去翻文献，你很可能会遇到这个设定下的标准数学形式化：",[315,14785,14786],{},"MDP（Markov Decision Process，马尔可夫决策过程）","。一个 MDP 是五元组 ",[367,14789,14791,14832],{"className":14790},[370],[367,14792,14794],{"className":14793},[374],[376,14795,14796],{"xmlns":378},[380,14797,14798,14829],{},[383,14799,14800,14803,14806,14808,14810,14812,14814,14816,14818,14820,14826],{},[621,14801,14802],{"stretchy":628},"⟨",[386,14804,14805],{},"S",[621,14807,2656],{"separator":397},[386,14809,14308],{},[621,14811,2656],{"separator":397},[386,14813,5734],{},[621,14815,2656],{"separator":397},[386,14817,1895],{},[621,14819,2656],{"separator":397},[613,14821,14822,14824],{},[386,14823,4724],{},[2932,14825,1083],{},[621,14827,14828],{"stretchy":628},"⟩",[390,14830,14831],{"encoding":392},"\\langle S, A, R, P, \\rho_0 \\rangle",[367,14833,14835],{"className":14834,"ariaHidden":397},[396],[367,14836,14838,14841,14844,14848,14851,14854,14857,14860,14863,14866,14869,14872,14875,14878,14881,14921],{"className":14837},[401],[367,14839],{"className":14840,"style":726},[405],[367,14842,14802],{"className":14843},[733],[367,14845,14805],{"className":14846,"style":14847},[410,411],"margin-right:0.0576em;",[367,14849,2656],{"className":14850},[2685],[367,14852],{"className":14853,"style":1773},[711],[367,14855,14308],{"className":14856},[410,411],[367,14858,2656],{"className":14859},[2685],[367,14861],{"className":14862,"style":1773},[711],[367,14864,5734],{"className":14865,"style":5749},[410,411],[367,14867,2656],{"className":14868},[2685],[367,14870],{"className":14871,"style":1773},[711],[367,14873,1895],{"className":14874,"style":1922},[410,411],[367,14876,2656],{"className":14877},[2685],[367,14879],{"className":14880,"style":1773},[711],[367,14882,14884,14887],{"className":14883},[410],[367,14885,4724],{"className":14886},[410,411],[367,14888,14890],{"className":14889},[661],[367,14891,14893,14913],{"className":14892},[665,666],[367,14894,14896,14910],{"className":14895},[670],[367,14897,14899],{"className":14898,"style":4456},[674],[367,14900,14901,14904],{"style":678},[367,14902],{"className":14903,"style":683},[682],[367,14905,14907],{"className":14906},[687,688,689,690],[367,14908,1083],{"className":14909},[410,690],[367,14911,698],{"className":14912},[697],[367,14914,14916],{"className":14915},[670],[367,14917,14919],{"className":14918,"style":705},[674],[367,14920],{},[367,14922,14828],{"className":14923},[777],"，其中：",[233,14926,14927,14958,14989,15384,15719],{},[236,14928,14929,14957],{},[367,14930,14932,14945],{"className":14931},[370],[367,14933,14935],{"className":14934},[374],[376,14936,14937],{"xmlns":378},[380,14938,14939,14943],{},[383,14940,14941],{},[386,14942,14805],{},[390,14944,14805],{"encoding":392},[367,14946,14948],{"className":14947,"ariaHidden":397},[396],[367,14949,14951,14954],{"className":14950},[401],[367,14952],{"className":14953,"style":2333},[405],[367,14955,14805],{"className":14956,"style":14847},[410,411]," 是所有合法状态的集合；",[236,14959,14960,14988],{},[367,14961,14963,14976],{"className":14962},[370],[367,14964,14966],{"className":14965},[374],[376,14967,14968],{"xmlns":378},[380,14969,14970,14974],{},[383,14971,14972],{},[386,14973,14308],{},[390,14975,14308],{"encoding":392},[367,14977,14979],{"className":14978,"ariaHidden":397},[396],[367,14980,14982,14985],{"className":14981},[401],[367,14983],{"className":14984,"style":2333},[405],[367,14986,14308],{"className":14987},[410,411]," 是所有合法动作的集合；",[236,14990,14991,15112,15113,15383],{},[367,14992,14994,15027],{"className":14993},[370],[367,14995,14997],{"className":14996},[374],[376,14998,14999],{"xmlns":378},[380,15000,15001,15024],{},[383,15002,15003,15005,15008,15010,15013,15015,15017,15019,15022],{},[386,15004,5734],{},[621,15006,15007],{},":",[386,15009,14805],{},[621,15011,15012],{},"×",[386,15014,14308],{},[621,15016,15012],{},[386,15018,14805],{},[621,15020,15021],{},"→",[386,15023,5734],{"mathvariant":7703},[390,15025,15026],{"encoding":392},"R : S \\times A \\times S \\to \\mathbb{R}",[367,15028,15030,15048,15067,15085,15103],{"className":15029,"ariaHidden":397},[396],[367,15031,15033,15036,15039,15042,15045],{"className":15032},[401],[367,15034],{"className":15035,"style":2333},[405],[367,15037,5734],{"className":15038,"style":5749},[410,411],[367,15040],{"className":15041,"style":712},[711],[367,15043,15007],{"className":15044},[716],[367,15046],{"className":15047,"style":712},[711],[367,15049,15051,15055,15058,15061,15064],{"className":15050},[401],[367,15052],{"className":15053,"style":15054},[405],"height:0.7667em;vertical-align:-0.0833em;",[367,15056,14805],{"className":15057,"style":14847},[410,411],[367,15059],{"className":15060,"style":3132},[711],[367,15062,15012],{"className":15063},[3136],[367,15065],{"className":15066,"style":3132},[711],[367,15068,15070,15073,15076,15079,15082],{"className":15069},[401],[367,15071],{"className":15072,"style":15054},[405],[367,15074,14308],{"className":15075},[410,411],[367,15077],{"className":15078,"style":3132},[711],[367,15080,15012],{"className":15081},[3136],[367,15083],{"className":15084,"style":3132},[711],[367,15086,15088,15091,15094,15097,15100],{"className":15087},[401],[367,15089],{"className":15090,"style":2333},[405],[367,15092,14805],{"className":15093,"style":14847},[410,411],[367,15095],{"className":15096,"style":712},[711],[367,15098,15021],{"className":15099},[716],[367,15101],{"className":15102,"style":712},[711],[367,15104,15106,15109],{"className":15105},[401],[367,15107],{"className":15108,"style":7876},[405],[367,15110,5734],{"className":15111},[410,7909]," 是奖励函数，",[367,15114,15116,15169],{"className":15115},[370],[367,15117,15119],{"className":15118},[374],[376,15120,15121],{"xmlns":378},[380,15122,15123,15167],{},[383,15124,15125,15131,15133,15135,15137,15143,15145,15151,15153,15165],{},[613,15126,15127,15129],{},[386,15128,5770],{},[386,15130,619],{},[621,15132,623],{},[386,15134,5734],{},[621,15136,629],{"stretchy":628},[613,15138,15139,15141],{},[386,15140,388],{},[386,15142,619],{},[621,15144,2656],{"separator":397},[613,15146,15147,15149],{},[386,15148,216],{},[386,15150,619],{},[621,15152,2656],{"separator":397},[613,15154,15155,15157],{},[386,15156,388],{},[383,15158,15159,15161,15163],{},[386,15160,619],{},[621,15162,3029],{},[2932,15164,3603],{},[621,15166,638],{"stretchy":628},[390,15168,5811],{"encoding":392},[367,15170,15172,15227],{"className":15171,"ariaHidden":397},[396],[367,15173,15175,15178,15218,15221,15224],{"className":15174},[401],[367,15176],{"className":15177,"style":651},[405],[367,15179,15181,15184],{"className":15180},[410],[367,15182,5770],{"className":15183,"style":1021},[410,411],[367,15185,15187],{"className":15186},[661],[367,15188,15190,15210],{"className":15189},[665,666],[367,15191,15193,15207],{"className":15192},[670],[367,15194,15196],{"className":15195,"style":675},[674],[367,15197,15198,15201],{"style":5841},[367,15199],{"className":15200,"style":683},[682],[367,15202,15204],{"className":15203},[687,688,689,690],[367,15205,619],{"className":15206},[410,411,690],[367,15208,698],{"className":15209},[697],[367,15211,15213],{"className":15212},[670],[367,15214,15216],{"className":15215,"style":705},[674],[367,15217],{},[367,15219],{"className":15220,"style":712},[711],[367,15222,623],{"className":15223},[716],[367,15225],{"className":15226,"style":712},[711],[367,15228,15230,15233,15236,15239,15279,15282,15285,15325,15328,15331,15380],{"className":15229},[401],[367,15231],{"className":15232,"style":726},[405],[367,15234,5734],{"className":15235,"style":5749},[410,411],[367,15237,629],{"className":15238},[733],[367,15240,15242,15245],{"className":15241},[410],[367,15243,388],{"className":15244},[410,411],[367,15246,15248],{"className":15247},[661],[367,15249,15251,15271],{"className":15250},[665,666],[367,15252,15254,15268],{"className":15253},[670],[367,15255,15257],{"className":15256,"style":675},[674],[367,15258,15259,15262],{"style":678},[367,15260],{"className":15261,"style":683},[682],[367,15263,15265],{"className":15264},[687,688,689,690],[367,15266,619],{"className":15267},[410,411,690],[367,15269,698],{"className":15270},[697],[367,15272,15274],{"className":15273},[670],[367,15275,15277],{"className":15276,"style":705},[674],[367,15278],{},[367,15280,2656],{"className":15281},[2685],[367,15283],{"className":15284,"style":1773},[711],[367,15286,15288,15291],{"className":15287},[410],[367,15289,216],{"className":15290},[410,411],[367,15292,15294],{"className":15293},[661],[367,15295,15297,15317],{"className":15296},[665,666],[367,15298,15300,15314],{"className":15299},[670],[367,15301,15303],{"className":15302,"style":675},[674],[367,15304,15305,15308],{"style":678},[367,15306],{"className":15307,"style":683},[682],[367,15309,15311],{"className":15310},[687,688,689,690],[367,15312,619],{"className":15313},[410,411,690],[367,15315,698],{"className":15316},[697],[367,15318,15320],{"className":15319},[670],[367,15321,15323],{"className":15322,"style":705},[674],[367,15324],{},[367,15326,2656],{"className":15327},[2685],[367,15329],{"className":15330,"style":1773},[711],[367,15332,15334,15337],{"className":15333},[410],[367,15335,388],{"className":15336},[410,411],[367,15338,15340],{"className":15339},[661],[367,15341,15343,15372],{"className":15342},[665,666],[367,15344,15346,15369],{"className":15345},[670],[367,15347,15349],{"className":15348,"style":4456},[674],[367,15350,15351,15354],{"style":678},[367,15352],{"className":15353,"style":683},[682],[367,15355,15357],{"className":15356},[687,688,689,690],[367,15358,15360,15363,15366],{"className":15359},[410,690],[367,15361,619],{"className":15362},[410,411,690],[367,15364,3029],{"className":15365},[3136,690],[367,15367,3603],{"className":15368},[410,690],[367,15370,698],{"className":15371},[697],[367,15373,15375],{"className":15374},[670],[367,15376,15378],{"className":15377,"style":5172},[674],[367,15379],{},[367,15381,638],{"className":15382},[777],"；",[236,15385,15386,15497,15498,15598,15599,15627,15628,15656,15657,15718],{},[367,15387,15389,15421],{"className":15388},[370],[367,15390,15392],{"className":15391},[374],[376,15393,15394],{"xmlns":378},[380,15395,15396,15418],{},[383,15397,15398,15400,15402,15404,15406,15408,15410,15412,15414,15416],{},[386,15399,1895],{},[621,15401,15007],{},[386,15403,14805],{},[621,15405,15012],{},[386,15407,14308],{},[621,15409,15021],{},[386,15411,1895],{"mathvariant":2927},[621,15413,629],{"stretchy":628},[386,15415,14805],{},[621,15417,638],{"stretchy":628},[390,15419,15420],{"encoding":392},"P : S \\times A \\to \\mathcal{P}(S)",[367,15422,15424,15442,15460,15478],{"className":15423,"ariaHidden":397},[396],[367,15425,15427,15430,15433,15436,15439],{"className":15426},[401],[367,15428],{"className":15429,"style":2333},[405],[367,15431,1895],{"className":15432,"style":1922},[410,411],[367,15434],{"className":15435,"style":712},[711],[367,15437,15007],{"className":15438},[716],[367,15440],{"className":15441,"style":712},[711],[367,15443,15445,15448,15451,15454,15457],{"className":15444},[401],[367,15446],{"className":15447,"style":15054},[405],[367,15449,14805],{"className":15450,"style":14847},[410,411],[367,15452],{"className":15453,"style":3132},[711],[367,15455,15012],{"className":15456},[3136],[367,15458],{"className":15459,"style":3132},[711],[367,15461,15463,15466,15469,15472,15475],{"className":15462},[401],[367,15464],{"className":15465,"style":2333},[405],[367,15467,14308],{"className":15468},[410,411],[367,15470],{"className":15471,"style":712},[711],[367,15473,15021],{"className":15474},[716],[367,15476],{"className":15477,"style":712},[711],[367,15479,15481,15484,15488,15491,15494],{"className":15480},[401],[367,15482],{"className":15483,"style":726},[405],[367,15485,1895],{"className":15486,"style":15487},[410,2974],"margin-right:0.0822em;",[367,15489,629],{"className":15490},[733],[367,15492,14805],{"className":15493,"style":14847},[410,411],[367,15495,638],{"className":15496},[777]," 是转移概率函数（transition probability function），",[367,15499,15501,15533],{"className":15500},[370],[367,15502,15504],{"className":15503},[374],[376,15505,15506],{"xmlns":378},[380,15507,15508,15530],{},[383,15509,15510,15512,15514,15520,15522,15524,15526,15528],{},[386,15511,1895],{},[621,15513,629],{"stretchy":628},[3653,15515,15516,15518],{},[386,15517,388],{},[621,15519,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[386,15521,846],{"mathvariant":845},[386,15523,388],{},[621,15525,2656],{"separator":397},[386,15527,216],{},[621,15529,638],{"stretchy":628},[390,15531,15532],{"encoding":392},"P(s'|s,a)",[367,15534,15536],{"className":15535,"ariaHidden":397},[396],[367,15537,15539,15542,15545,15548,15580,15583,15586,15589,15592,15595],{"className":15538},[401],[367,15540],{"className":15541,"style":13120},[405],[367,15543,1895],{"className":15544,"style":1922},[410,411],[367,15546,629],{"className":15547},[733],[367,15549,15551,15554],{"className":15550},[410],[367,15552,388],{"className":15553},[410,411],[367,15555,15557],{"className":15556},[661],[367,15558,15560],{"className":15559},[665],[367,15561,15563],{"className":15562},[670],[367,15564,15566],{"className":15565,"style":12575},[674],[367,15567,15568,15571],{"style":4176},[367,15569],{"className":15570,"style":683},[682],[367,15572,15574],{"className":15573},[687,688,689,690],[367,15575,15577],{"className":15576},[410,690],[367,15578,11537],{"className":15579},[410,690],[367,15581,846],{"className":15582},[410],[367,15584,388],{"className":15585},[410,411],[367,15587,2656],{"className":15588},[2685],[367,15590],{"className":15591,"style":1773},[711],[367,15593,216],{"className":15594},[410,411],[367,15596,638],{"className":15597},[777]," 表示从状态 ",[367,15600,15602,15615],{"className":15601},[370],[367,15603,15605],{"className":15604},[374],[376,15606,15607],{"xmlns":378},[380,15608,15609,15613],{},[383,15610,15611],{},[386,15612,388],{},[390,15614,388],{"encoding":392},[367,15616,15618],{"className":15617,"ariaHidden":397},[396],[367,15619,15621,15624],{"className":15620},[401],[367,15622],{"className":15623,"style":406},[405],[367,15625,388],{"className":15626},[410,411]," 出发采取动作 ",[367,15629,15631,15644],{"className":15630},[370],[367,15632,15634],{"className":15633},[374],[376,15635,15636],{"xmlns":378},[380,15637,15638,15642],{},[383,15639,15640],{},[386,15641,216],{},[390,15643,216],{"encoding":392},[367,15645,15647],{"className":15646,"ariaHidden":397},[396],[367,15648,15650,15653],{"className":15649},[401],[367,15651],{"className":15652,"style":406},[405],[367,15654,216],{"className":15655},[410,411]," 后转移到状态 ",[367,15658,15660,15677],{"className":15659},[370],[367,15661,15663],{"className":15662},[374],[376,15664,15665],{"xmlns":378},[380,15666,15667,15675],{},[383,15668,15669],{},[3653,15670,15671,15673],{},[386,15672,388],{},[621,15674,11537],{"mathvariant":845,"lspace":1073,"rspace":1073},[390,15676,12770],{"encoding":392},[367,15678,15680],{"className":15679,"ariaHidden":397},[396],[367,15681,15683,15686],{"className":15682},[401],[367,15684],{"className":15685,"style":12575},[405],[367,15687,15689,15692],{"className":15688},[410],[367,15690,388],{"className":15691},[410,411],[367,15693,15695],{"className":15694},[661],[367,15696,15698],{"className":15697},[665],[367,15699,15701],{"className":15700},[670],[367,15702,15704],{"className":15703,"style":12575},[674],[367,15705,15706,15709],{"style":4176},[367,15707],{"className":15708,"style":683},[682],[367,15710,15712],{"className":15711},[687,688,689,690],[367,15713,15715],{"className":15714},[410,690],[367,15716,11537],{"className":15717},[410,690]," 的概率；",[236,15720,15721,15790],{},[367,15722,15724,15741],{"className":15723},[370],[367,15725,15727],{"className":15726},[374],[376,15728,15729],{"xmlns":378},[380,15730,15731,15739],{},[383,15732,15733],{},[613,15734,15735,15737],{},[386,15736,4724],{},[2932,15738,1083],{},[390,15740,4729],{"encoding":392},[367,15742,15744],{"className":15743,"ariaHidden":397},[396],[367,15745,15747,15750],{"className":15746},[401],[367,15748],{"className":15749,"style":590},[405],[367,15751,15753,15756],{"className":15752},[410],[367,15754,4724],{"className":15755},[410,411],[367,15757,15759],{"className":15758},[661],[367,15760,15762,15782],{"className":15761},[665,666],[367,15763,15765,15779],{"className":15764},[670],[367,15766,15768],{"className":15767,"style":4456},[674],[367,15769,15770,15773],{"style":678},[367,15771],{"className":15772,"style":683},[682],[367,15774,15776],{"className":15775},[687,688,689,690],[367,15777,1083],{"className":15778},[410,690],[367,15780,698],{"className":15781},[697],[367,15783,15785],{"className":15784},[670],[367,15786,15788],{"className":15787,"style":705},[674],[367,15789],{}," 是初始状态分布。",[212,15792,15793,15794,15799],{},"「马尔可夫决策过程」这个名字来自系统服从",[216,15795,15798],{"href":15796,"rel":15797},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FMarkov_property",[220],"马尔可夫性质（Markov property）","这一事实：转移只依赖最近的状态和动作，不依赖更早的历史。",[15801,15802,15803],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":40,"searchDepth":1584,"depth":1590,"links":15805},[15806,15807,15822],{"id":256,"depth":1584,"text":257},{"id":301,"depth":1584,"text":301,"children":15808},[15809,15810,15811,15815,15816,15817,15818,15819,15820,15821],{"id":359,"depth":1590,"text":359},{"id":535,"depth":1590,"text":535},{"id":556,"depth":1590,"text":556,"children":15812},[15813,15814],{"id":1555,"depth":1596,"text":1555},{"id":1685,"depth":1596,"text":1685},{"id":4307,"depth":1590,"text":4307},{"id":5716,"depth":1590,"text":5716},{"id":6978,"depth":1590,"text":6979},{"id":6974,"depth":1590,"text":6974},{"id":10681,"depth":1590,"text":10682},{"id":11432,"depth":1590,"text":11432},{"id":14248,"depth":1590,"text":14248},{"id":14779,"depth":1584,"text":14780},"md",{},{"title":15826,"description":23},[6],"posts\u002Fspinning-up-rl-intro",[11,12,13,14],"S8OluMHpUVgtFD6nwlsqs8PDOVYVvgHIiyhrxJfvsEg",1790607246384]