[{"data":1,"prerenderedAt":49257},["ShallowReactive",2],{"blog-posts":3,"content-\u002Fposts\u002Fllm-memory-compute-tradeoff":143},[4,17,25,32,41,49,55,61,66,71,78,85,93,99,106,114,120,126,132,138],{"_path":5,"title":6,"description":7,"date":8,"categories":9,"tags":10,"draft":16},"\u002Fposts\u002Fllm-memory-compute-tradeoff","现代 LLM 的 Memory-Compute Trade-off：从 MHA 到 MLA、Sparse 与 Linear Attention","ds4.1f 上线了。它的架构 novel 在哪，便宜和快又是从哪省出来的。","2026-09-11 12:00:00","技术杂谈",[11,12,13,14,15],"LLMs","Attention","MLA","DeepSeek","长上下文",false,{"_path":18,"title":19,"description":20,"date":21,"categories":22,"tags":23,"draft":16},"\u002Fposts\u002Fwhy-we-still-need-to-write-a-blog","为什么还需要手写博客？","AI这么强，还要古法写博客吗？","2026-09-01 13:17:53","生活杂谈",[24],"博客",{"_path":26,"title":27,"description":28,"date":29,"categories":30,"tags":31,"draft":16},"\u002Fposts\u002Finteresting-music-rhythm","有趣的音乐节奏","","2026-07-16 12:47:24","音乐",null,{"_path":33,"title":34,"description":35,"date":36,"categories":37,"tags":38,"draft":16},"\u002Fposts\u002Ftowards-monosemanticity","LLM 可解释性（一）：SAE与单语义","读 Anthropic《Towards Monosemanticity》。从难以解释的神经元出发, 讲清superposition为什么让单个神经元混成一团, SAE怎么把多语义的激活分解成单语义特征。","2026-07-12 22:00:00","技术",[11,39,40],"可解释性","SAE",{"_path":42,"title":43,"description":28,"date":44,"categories":37,"tags":45,"draft":16},"\u002Fposts\u002Fclaude-code-settings","Claude Code 配置分享","2026-06-23 14:22:12",[11,46,47,48],"AI Agent","Agent","Claude Code",{"_path":50,"title":51,"description":52,"date":53,"categories":9,"tags":54,"draft":16},"\u002Fposts\u002Fmy-thoughts-on-the-ai-industry","简单聊聊对 AI Agent 行业的理解","写这篇文章的初衷，是我发觉逐渐忘了前两个月的 AI\u002FLLMs 是什么样的能力和情况了。意识中似乎某些在突然加速，但理性和直觉告诉我这不对，作为行内人，我知道这是一个渐进的、累积的过程。我想自己回忆一下，Agent的发展。","2026-06-20 13:14:25",[11,46,47],{"_path":56,"title":57,"description":28,"date":58,"categories":37,"tags":59,"draft":16},"\u002Fposts\u002Frotational-positional-encoding","RoPE","2026-06-17 18:31:40",[11,60,57],"Transformer",{"_path":62,"title":63,"description":28,"date":64,"categories":37,"tags":65,"draft":16},"\u002Fposts\u002Fintroduction-to-large-language-model","大语言模型入门基础","2026-06-16 22:36:11",[60,11],{"_path":67,"title":68,"description":28,"date":69,"categories":9,"tags":70,"draft":16},"\u002Fposts\u002Fthought-about-coding-agent","关于 Coding Agent 使用的遐思","2026-06-03 17:54:20",[47],{"_path":72,"title":73,"description":74,"date":75,"categories":37,"tags":76},"\u002Fposts\u002Fabout-ssh-usage","关于SSH的使用——客户端篇","介绍SSH客户端的用法。","2025-06-17 02:44:57",[77],"工具",{"_path":79,"title":80,"description":81,"date":82,"categories":9,"tags":83},"\u002Fposts\u002Fopensourcesoftwarestarttutorial","如何开始一个开源项目？","浅谈我对开源项目的认知。","2025-04-17 18:00:00",[84],"开源",{"_path":86,"title":87,"description":88,"date":89,"categories":90,"tags":91},"\u002Fposts\u002Fmotivation","渴求动力","有意识的生活并主动地行动会使得生活更美好。","2025-03-05 00:44:57","生活-情感",[92],"随笔",{"_path":94,"title":95,"description":96,"date":97,"categories":90,"tags":98},"\u002Fposts\u002F2024summary","2024年终总结","啊，痛彻的时光；啊，悲怆的人！","2025-01-01 01:00:00",[92],{"_path":100,"title":101,"description":102,"date":103,"categories":37,"tags":104},"\u002Fposts\u002Fvps-crash-startup","快速搭建个人VPS","如何快速搭建个人VPS。","2024-12-31 19:00:00",[105],"VPS",{"_path":107,"title":108,"description":109,"date":110,"categories":37,"tags":111},"\u002Fposts\u002Fdata-label-software","AI数据（自动）标注软件概述","对市面上目前的AI数据（自动）标注软件进行调研。","2024-12-20 14:00:00",[112,113],"数据标注","深度学习",{"_path":115,"title":116,"description":117,"date":118,"categories":90,"tags":119},"\u002Fposts\u002Fself-destruction","生活的崩坏与自我毁灭","是什么在牵引着我？我的行为，我的思想？我的感受？我所处的地方？它们在何处？","2024-12-19 12:00:00",[92],{"_path":121,"title":122,"description":123,"date":124,"categories":37,"tags":125},"\u002Fposts\u002Fcs194-196_llmagents","LLM Agents 课程笔记","UCB CS294\u002F194-196 Large Language Model Agents 的课程笔记。","2024-11-30 20:00:00",[47],{"_path":127,"title":128,"description":129,"date":130,"categories":90,"tags":131},"\u002Fposts\u002Fwhatsyourworth","价值、位置","你们的生存究竟有何价值？如果毫无价值，你们究竟为何存在？","2024-11-29 02:00:00",[92],{"_path":133,"title":134,"description":135,"date":136,"categories":90,"tags":137},"\u002Fposts\u002Fblogrelifever2","重启博客v2","忙里偷闲学习前端，还是决定结合想法手搓一个自己的博客。","2024-11-25 00:00:00",[24],{"_path":139,"title":140,"description":141,"date":142,"categories":90,"tags":31},"\u002Fposts\u002Fblogrelife","重启博客","在我的表达欲与现实所需(问题解决记录)驱动下，我决定重写博客。","2024-10-21 20:44:57",{"_path":5,"_dir":144,"_draft":16,"_partial":16,"_locale":28,"title":6,"description":7,"date":8,"categories":9,"tags":145,"draft":16,"body":146,"_type":49251,"_id":49252,"_source":49253,"_file":49254,"_stem":49255,"_extension":49256},"posts",[11,12,13,14,15],{"type":147,"children":148,"toc":49233},"root",[149,163,168,371,378,1180,1740,3374,4854,5525,5813,5995,6196,6201,6206,6212,6237,6249,6261,6266,6697,7154,7439,7811,7816,7821,7827,8020,8025,8030,8124,8247,8502,8632,9355,9677,9682,9844,9860,9987,9995,10007,10012,10017,10171,11126,11340,12565,13277,13282,14258,14450,15625,17092,17430,17435,17440,18444,18726,20132,20684,21084,21659,21667,22367,23285,23290,23575,23580,23822,23827,23832,25111,25116,25627,25730,26248,26295,26926,26982,26992,26997,27183,27188,27193,27243,27769,27860,28294,28298,28883,28888,28893,28916,29380,29390,29401,29406,29411,29681,29946,29951,30061,30071,30076,30504,30597,30748,30754,30761,31190,31198,31203,31294,31299,31332,31337,31342,31557,31659,31774,31978,31984,32138,32217,32222,32369,32374,32380,32392,32912,32917,33356,33693,33769,33954,34311,34316,34321,34329,34520,34532,34623,34628,34634,34639,34703,34766,34772,34777,34832,34837,34898,34903,35149,35168,35173,35178,35255,35412,35575,35580,35586,35591,35773,35907,35980,36065,36162,36167,36297,36441,36587,36823,36828,36995,37175,37180,37185,37330,37335,37432,37534,38002,38007,38021,38065,38202,38324,38373,38413,38418,38428,38433,38986,39000,39012,39024,39029,39106,39111,39116,39348,39353,39518,39687,39978,39983,39993,39998,40116,40378,40383,40444,40452,40457,40462,40467,40670,40680,40685,40926,40931,40936,40944,40949,40954,41345,41350,41931,41936,42270,42531,42658,42663,42927,42935,43263,43364,43449,43459,43464,43469,43572,43577,43587,43837,43847,44010,44015,44224,44229,44514,44524,44529,44534,44546,44691,44696,44706,44711,44717,44722,44736,44750,44755,44760,44857,44954,45051,45063,45162,45167,45271,45276,45602,45607,45828,45833,45992,46004,46263,46268,46319,46324,46559,46564,46713,46718,46723,46849,46854,47087,47092,47201,47206,47371,47376,47502,47507,47633,47638,47764,47769,47895,47905,47910,47915,47920,48080,48085,48090,48095,48231,48236,48366,48491,48617,48733,48858,48980,48985,49177,49182,49187,49197,49207,49217,49222,49227],{"type":150,"tag":151,"props":152,"children":153},"element","p",{},[154,157],{"type":155,"value":156},"text","GPT6出来之后，体感上很多人都很焦虑迷茫，真的到了AGI time，接下来该做什么呢？虽然说Astra用下来，在一些case上还是会overfit，但真的已经能胜任大多数人完成的任务了（在数字世界里）。这两天ds4.1f正式上线了，如此novel的架构让我感到新奇，模型架构与硬件的co-design达到一定水平的杰作，让我们来看一下吧。",{"type":150,"tag":158,"props":159,"children":160},"strong",{},[161],{"type":155,"value":162},"这篇由 AI 整理。",{"type":150,"tag":151,"props":164,"children":165},{},[166],{"type":155,"value":167},"路线大概是这样：从 MHA 出发，一路看到 Linear Attention，看每一代分别在砍哪一项成本。",{"type":150,"tag":169,"props":170,"children":173},"span",{"className":171},[172],"katex-display",[174],{"type":150,"tag":169,"props":175,"children":178},{"className":176},[177],"katex",[179,229],{"type":150,"tag":169,"props":180,"children":183},{"className":181},[182],"katex-mathml",[184],{"type":150,"tag":185,"props":186,"children":189},"math",{"xmlns":187,"display":188},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML","block",[190],{"type":150,"tag":191,"props":192,"children":193},"semantics",{},[194,222],{"type":150,"tag":195,"props":196,"children":197},"mrow",{},[198],{"type":150,"tag":199,"props":200,"children":202},"menclose",{"notation":201},"box",[203],{"type":150,"tag":204,"props":205,"children":208},"mstyle",{"scriptlevel":206,"displaystyle":207},"0","false",[209],{"type":150,"tag":204,"props":210,"children":211},{"scriptlevel":206,"displaystyle":207},[212],{"type":150,"tag":204,"props":213,"children":215},{"scriptlevel":206,"displaystyle":214},"true",[216],{"type":150,"tag":217,"props":218,"children":219},"mtext",{},[220],{"type":155,"value":221},"现代 LLM 架构的主线，是不断消除 memory 和 compute 的冗余。",{"type":150,"tag":223,"props":224,"children":226},"annotation",{"encoding":225},"application\u002Fx-tex",[227],{"type":155,"value":228},"\\boxed{\\text{现代 LLM 架构的主线，是不断消除 memory 和 compute 的冗余。}}",{"type":150,"tag":169,"props":230,"children":233},{"className":231,"ariaHidden":214},[232],"katex-html",[234],{"type":150,"tag":169,"props":235,"children":238},{"className":236},[237],"base",[239,245],{"type":150,"tag":169,"props":240,"children":244},{"className":241,"style":243},[242],"strut","height:1.5578em;vertical-align:-0.5344em;",[],{"type":150,"tag":169,"props":246,"children":249},{"className":247},[248],"mord",[250],{"type":150,"tag":169,"props":251,"children":255},{"className":252},[253,254],"vlist-t","vlist-t2",[256,359],{"type":150,"tag":169,"props":257,"children":260},{"className":258},[259],"vlist-r",[261,352],{"type":150,"tag":169,"props":262,"children":266},{"className":263,"style":265},[264],"vlist","height:1.0233em;",[267,337],{"type":150,"tag":169,"props":268,"children":270},{"style":269},"top:-3.5578em;",[271,277],{"type":150,"tag":169,"props":272,"children":276},{"className":273,"style":275},[274],"pstrut","height:3.5578em;",[],{"type":150,"tag":169,"props":278,"children":281},{"className":279},[280],"boxpad",[282],{"type":150,"tag":169,"props":283,"children":285},{"className":284},[248],[286],{"type":150,"tag":169,"props":287,"children":289},{"className":288},[248],[290],{"type":150,"tag":169,"props":291,"children":293},{"className":292},[248,155],[294,301,307,313,319,325,331],{"type":150,"tag":169,"props":295,"children":298},{"className":296},[248,297],"cjk_fallback",[299],{"type":155,"value":300},"现代",{"type":150,"tag":169,"props":302,"children":304},{"className":303},[248],[305],{"type":155,"value":306}," LLM ",{"type":150,"tag":169,"props":308,"children":310},{"className":309},[248,297],[311],{"type":155,"value":312},"架构的主线，是不断消除",{"type":150,"tag":169,"props":314,"children":316},{"className":315},[248],[317],{"type":155,"value":318}," memory ",{"type":150,"tag":169,"props":320,"children":322},{"className":321},[248,297],[323],{"type":155,"value":324},"和",{"type":150,"tag":169,"props":326,"children":328},{"className":327},[248],[329],{"type":155,"value":330}," compute ",{"type":150,"tag":169,"props":332,"children":334},{"className":333},[248,297],[335],{"type":155,"value":336},"的冗余。",{"type":150,"tag":169,"props":338,"children":340},{"style":339},"top:-3.0233em;",[341,345],{"type":150,"tag":169,"props":342,"children":344},{"className":343,"style":275},[274],[],{"type":150,"tag":169,"props":346,"children":351},{"className":347,"style":350},[348,349],"stretchy","fbox","height:1.5578em;border-style:solid;border-width:0.04em;",[],{"type":150,"tag":169,"props":353,"children":356},{"className":354},[355],"vlist-s",[357],{"type":155,"value":358},"​",{"type":150,"tag":169,"props":360,"children":362},{"className":361},[259],[363],{"type":150,"tag":169,"props":364,"children":367},{"className":365,"style":366},[264],"height:0.5344em;",[368],{"type":150,"tag":169,"props":369,"children":370},{},[],{"type":150,"tag":372,"props":373,"children":375},"h2",{"id":374},"mha-与-kv-cache",[376],{"type":155,"value":377},"MHA 与 KV Cache",{"type":150,"tag":151,"props":379,"children":380},{},[381,383,709,711,903,905,949,951,1018,1020,1178],{"type":155,"value":382},"MHA（Multi-Head Attention）是这一切的起点，2017 年《Attention Is All You Need》里提出来的。数学上它就是多个独立的单头注意力拼起来。假设输入向量序列是 ",{"type":150,"tag":169,"props":384,"children":386},{"className":385},[177],[387,469],{"type":150,"tag":169,"props":388,"children":390},{"className":389},[182],[391],{"type":150,"tag":185,"props":392,"children":393},{"xmlns":187},[394],{"type":150,"tag":191,"props":395,"children":396},{},[397,464],{"type":150,"tag":195,"props":398,"children":399},{},[400,416,422,434,438,443,448,452],{"type":150,"tag":401,"props":402,"children":403},"msub",{},[404,410],{"type":150,"tag":405,"props":406,"children":407},"mi",{},[408],{"type":155,"value":409},"x",{"type":150,"tag":411,"props":412,"children":413},"mn",{},[414],{"type":155,"value":415},"1",{"type":150,"tag":417,"props":418,"children":419},"mo",{"separator":214},[420],{"type":155,"value":421},",",{"type":150,"tag":401,"props":423,"children":424},{},[425,429],{"type":150,"tag":405,"props":426,"children":427},{},[428],{"type":155,"value":409},{"type":150,"tag":411,"props":430,"children":431},{},[432],{"type":155,"value":433},"2",{"type":150,"tag":417,"props":435,"children":436},{"separator":214},[437],{"type":155,"value":421},{"type":150,"tag":417,"props":439,"children":440},{},[441],{"type":155,"value":442},"⋯",{"type":150,"tag":217,"props":444,"children":445},{},[446],{"type":155,"value":447}," ",{"type":150,"tag":417,"props":449,"children":450},{"separator":214},[451],{"type":155,"value":421},{"type":150,"tag":401,"props":453,"children":454},{},[455,459],{"type":150,"tag":405,"props":456,"children":457},{},[458],{"type":155,"value":409},{"type":150,"tag":405,"props":460,"children":461},{},[462],{"type":155,"value":463},"l",{"type":150,"tag":223,"props":465,"children":466},{"encoding":225},[467],{"type":155,"value":468},"x_1, x_2, \\cdots, x_l",{"type":150,"tag":169,"props":470,"children":472},{"className":471,"ariaHidden":214},[232],[473],{"type":150,"tag":169,"props":474,"children":476},{"className":475},[237],[477,482,549,555,561,618,623,627,633,637,641,646,650],{"type":150,"tag":169,"props":478,"children":481},{"className":479,"style":480},[242],"height:0.625em;vertical-align:-0.1944em;",[],{"type":150,"tag":169,"props":483,"children":485},{"className":484},[248],[486,492],{"type":150,"tag":169,"props":487,"children":490},{"className":488},[248,489],"mathnormal",[491],{"type":155,"value":409},{"type":150,"tag":169,"props":493,"children":496},{"className":494},[495],"msupsub",[497],{"type":150,"tag":169,"props":498,"children":500},{"className":499},[253,254],[501,537],{"type":150,"tag":169,"props":502,"children":504},{"className":503},[259],[505,532],{"type":150,"tag":169,"props":506,"children":509},{"className":507,"style":508},[264],"height:0.3011em;",[510],{"type":150,"tag":169,"props":511,"children":513},{"style":512},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[514,519],{"type":150,"tag":169,"props":515,"children":518},{"className":516,"style":517},[274],"height:2.7em;",[],{"type":150,"tag":169,"props":520,"children":526},{"className":521},[522,523,524,525],"sizing","reset-size6","size3","mtight",[527],{"type":150,"tag":169,"props":528,"children":530},{"className":529},[248,525],[531],{"type":155,"value":415},{"type":150,"tag":169,"props":533,"children":535},{"className":534},[355],[536],{"type":155,"value":358},{"type":150,"tag":169,"props":538,"children":540},{"className":539},[259],[541],{"type":150,"tag":169,"props":542,"children":545},{"className":543,"style":544},[264],"height:0.15em;",[546],{"type":150,"tag":169,"props":547,"children":548},{},[],{"type":150,"tag":169,"props":550,"children":553},{"className":551},[552],"mpunct",[554],{"type":155,"value":421},{"type":150,"tag":169,"props":556,"children":560},{"className":557,"style":559},[558],"mspace","margin-right:0.1667em;",[],{"type":150,"tag":169,"props":562,"children":564},{"className":563},[248],[565,570],{"type":150,"tag":169,"props":566,"children":568},{"className":567},[248,489],[569],{"type":155,"value":409},{"type":150,"tag":169,"props":571,"children":573},{"className":572},[495],[574],{"type":150,"tag":169,"props":575,"children":577},{"className":576},[253,254],[578,607],{"type":150,"tag":169,"props":579,"children":581},{"className":580},[259],[582,602],{"type":150,"tag":169,"props":583,"children":585},{"className":584,"style":508},[264],[586],{"type":150,"tag":169,"props":587,"children":588},{"style":512},[589,593],{"type":150,"tag":169,"props":590,"children":592},{"className":591,"style":517},[274],[],{"type":150,"tag":169,"props":594,"children":596},{"className":595},[522,523,524,525],[597],{"type":150,"tag":169,"props":598,"children":600},{"className":599},[248,525],[601],{"type":155,"value":433},{"type":150,"tag":169,"props":603,"children":605},{"className":604},[355],[606],{"type":155,"value":358},{"type":150,"tag":169,"props":608,"children":610},{"className":609},[259],[611],{"type":150,"tag":169,"props":612,"children":614},{"className":613,"style":544},[264],[615],{"type":150,"tag":169,"props":616,"children":617},{},[],{"type":150,"tag":169,"props":619,"children":621},{"className":620},[552],[622],{"type":155,"value":421},{"type":150,"tag":169,"props":624,"children":626},{"className":625,"style":559},[558],[],{"type":150,"tag":169,"props":628,"children":631},{"className":629},[630],"minner",[632],{"type":155,"value":442},{"type":150,"tag":169,"props":634,"children":636},{"className":635,"style":559},[558],[],{"type":150,"tag":169,"props":638,"children":640},{"className":639,"style":559},[558],[],{"type":150,"tag":169,"props":642,"children":644},{"className":643},[552],[645],{"type":155,"value":421},{"type":150,"tag":169,"props":647,"children":649},{"className":648,"style":559},[558],[],{"type":150,"tag":169,"props":651,"children":653},{"className":652},[248],[654,659],{"type":150,"tag":169,"props":655,"children":657},{"className":656},[248,489],[658],{"type":155,"value":409},{"type":150,"tag":169,"props":660,"children":662},{"className":661},[495],[663],{"type":150,"tag":169,"props":664,"children":666},{"className":665},[253,254],[667,698],{"type":150,"tag":169,"props":668,"children":670},{"className":669},[259],[671,693],{"type":150,"tag":169,"props":672,"children":675},{"className":673,"style":674},[264],"height:0.3361em;",[676],{"type":150,"tag":169,"props":677,"children":678},{"style":512},[679,683],{"type":150,"tag":169,"props":680,"children":682},{"className":681,"style":517},[274],[],{"type":150,"tag":169,"props":684,"children":686},{"className":685},[522,523,524,525],[687],{"type":150,"tag":169,"props":688,"children":691},{"className":689,"style":690},[248,489,525],"margin-right:0.0197em;",[692],{"type":155,"value":463},{"type":150,"tag":169,"props":694,"children":696},{"className":695},[355],[697],{"type":155,"value":358},{"type":150,"tag":169,"props":699,"children":701},{"className":700},[259],[702],{"type":150,"tag":169,"props":703,"children":705},{"className":704,"style":544},[264],[706],{"type":150,"tag":169,"props":707,"children":708},{},[],{"type":155,"value":710},"，其中 ",{"type":150,"tag":169,"props":712,"children":714},{"className":713},[177],[715,765],{"type":150,"tag":169,"props":716,"children":718},{"className":717},[182],[719],{"type":150,"tag":185,"props":720,"children":721},{"xmlns":187},[722],{"type":150,"tag":191,"props":723,"children":724},{},[725,760],{"type":150,"tag":195,"props":726,"children":727},{},[728,740,745],{"type":150,"tag":401,"props":729,"children":730},{},[731,735],{"type":150,"tag":405,"props":732,"children":733},{},[734],{"type":155,"value":409},{"type":150,"tag":405,"props":736,"children":737},{},[738],{"type":155,"value":739},"i",{"type":150,"tag":417,"props":741,"children":742},{},[743],{"type":155,"value":744},"∈",{"type":150,"tag":746,"props":747,"children":748},"msup",{},[749,755],{"type":150,"tag":405,"props":750,"children":752},{"mathvariant":751},"double-struck",[753],{"type":155,"value":754},"R",{"type":150,"tag":405,"props":756,"children":757},{},[758],{"type":155,"value":759},"d",{"type":150,"tag":223,"props":761,"children":762},{"encoding":225},[763],{"type":155,"value":764},"x_i \\in \\mathbb{R}^d",{"type":150,"tag":169,"props":766,"children":768},{"className":767,"ariaHidden":214},[232],[769,851],{"type":150,"tag":169,"props":770,"children":772},{"className":771},[237],[773,778,836,841,847],{"type":150,"tag":169,"props":774,"children":777},{"className":775,"style":776},[242],"height:0.6891em;vertical-align:-0.15em;",[],{"type":150,"tag":169,"props":779,"children":781},{"className":780},[248],[782,787],{"type":150,"tag":169,"props":783,"children":785},{"className":784},[248,489],[786],{"type":155,"value":409},{"type":150,"tag":169,"props":788,"children":790},{"className":789},[495],[791],{"type":150,"tag":169,"props":792,"children":794},{"className":793},[253,254],[795,825],{"type":150,"tag":169,"props":796,"children":798},{"className":797},[259],[799,820],{"type":150,"tag":169,"props":800,"children":803},{"className":801,"style":802},[264],"height:0.3117em;",[804],{"type":150,"tag":169,"props":805,"children":806},{"style":512},[807,811],{"type":150,"tag":169,"props":808,"children":810},{"className":809,"style":517},[274],[],{"type":150,"tag":169,"props":812,"children":814},{"className":813},[522,523,524,525],[815],{"type":150,"tag":169,"props":816,"children":818},{"className":817},[248,489,525],[819],{"type":155,"value":739},{"type":150,"tag":169,"props":821,"children":823},{"className":822},[355],[824],{"type":155,"value":358},{"type":150,"tag":169,"props":826,"children":828},{"className":827},[259],[829],{"type":150,"tag":169,"props":830,"children":832},{"className":831,"style":544},[264],[833],{"type":150,"tag":169,"props":834,"children":835},{},[],{"type":150,"tag":169,"props":837,"children":840},{"className":838,"style":839},[558],"margin-right:0.2778em;",[],{"type":150,"tag":169,"props":842,"children":845},{"className":843},[844],"mrel",[846],{"type":155,"value":744},{"type":150,"tag":169,"props":848,"children":850},{"className":849,"style":839},[558],[],{"type":150,"tag":169,"props":852,"children":854},{"className":853},[237],[855,860],{"type":150,"tag":169,"props":856,"children":859},{"className":857,"style":858},[242],"height:0.8491em;",[],{"type":150,"tag":169,"props":861,"children":863},{"className":862},[248],[864,870],{"type":150,"tag":169,"props":865,"children":868},{"className":866},[248,867],"mathbb",[869],{"type":155,"value":754},{"type":150,"tag":169,"props":871,"children":873},{"className":872},[495],[874],{"type":150,"tag":169,"props":875,"children":877},{"className":876},[253],[878],{"type":150,"tag":169,"props":879,"children":881},{"className":880},[259],[882],{"type":150,"tag":169,"props":883,"children":885},{"className":884,"style":858},[264],[886],{"type":150,"tag":169,"props":887,"children":889},{"style":888},"top:-3.063em;margin-right:0.05em;",[890,894],{"type":150,"tag":169,"props":891,"children":893},{"className":892,"style":517},[274],[],{"type":150,"tag":169,"props":895,"children":897},{"className":896},[522,523,524,525],[898],{"type":150,"tag":169,"props":899,"children":901},{"className":900},[248,489,525],[902],{"type":155,"value":759},{"type":155,"value":904},"。",{"type":150,"tag":169,"props":906,"children":908},{"className":907},[177],[909,931],{"type":150,"tag":169,"props":910,"children":912},{"className":911},[182],[913],{"type":150,"tag":185,"props":914,"children":915},{"xmlns":187},[916],{"type":150,"tag":191,"props":917,"children":918},{},[919,927],{"type":150,"tag":195,"props":920,"children":921},{},[922],{"type":150,"tag":405,"props":923,"children":924},{},[925],{"type":155,"value":926},"h",{"type":150,"tag":223,"props":928,"children":929},{"encoding":225},[930],{"type":155,"value":926},{"type":150,"tag":169,"props":932,"children":934},{"className":933,"ariaHidden":214},[232],[935],{"type":150,"tag":169,"props":936,"children":938},{"className":937},[237],[939,944],{"type":150,"tag":169,"props":940,"children":943},{"className":941,"style":942},[242],"height:0.6944em;",[],{"type":150,"tag":169,"props":945,"children":947},{"className":946},[248,489],[948],{"type":155,"value":926},{"type":155,"value":950}," 是 head 数，",{"type":150,"tag":169,"props":952,"children":954},{"className":953},[177],[955,988],{"type":150,"tag":169,"props":956,"children":958},{"className":957},[182],[959],{"type":150,"tag":185,"props":960,"children":961},{"xmlns":187},[962],{"type":150,"tag":191,"props":963,"children":964},{},[965,983],{"type":150,"tag":195,"props":966,"children":967},{},[968,973,978],{"type":150,"tag":417,"props":969,"children":970},{"stretchy":207},[971],{"type":155,"value":972},"(",{"type":150,"tag":405,"props":974,"children":975},{},[976],{"type":155,"value":977},"s",{"type":150,"tag":417,"props":979,"children":980},{"stretchy":207},[981],{"type":155,"value":982},")",{"type":150,"tag":223,"props":984,"children":985},{"encoding":225},[986],{"type":155,"value":987},"(s)",{"type":150,"tag":169,"props":989,"children":991},{"className":990,"ariaHidden":214},[232],[992],{"type":150,"tag":169,"props":993,"children":995},{"className":994},[237],[996,1001,1007,1012],{"type":150,"tag":169,"props":997,"children":1000},{"className":998,"style":999},[242],"height:1em;vertical-align:-0.25em;",[],{"type":150,"tag":169,"props":1002,"children":1005},{"className":1003},[1004],"mopen",[1006],{"type":155,"value":972},{"type":150,"tag":169,"props":1008,"children":1010},{"className":1009},[248,489],[1011],{"type":155,"value":977},{"type":150,"tag":169,"props":1013,"children":1016},{"className":1014},[1015],"mclose",[1017],{"type":155,"value":982},{"type":155,"value":1019}," 是 head 的下标，",{"type":150,"tag":169,"props":1021,"children":1023},{"className":1022},[177],[1024,1083],{"type":150,"tag":169,"props":1025,"children":1027},{"className":1026},[182],[1028],{"type":150,"tag":185,"props":1029,"children":1030},{"xmlns":187},[1031],{"type":150,"tag":191,"props":1032,"children":1033},{},[1034,1078],{"type":150,"tag":195,"props":1035,"children":1036},{},[1037,1041,1046,1050,1054,1058,1062,1066,1070,1074],{"type":150,"tag":405,"props":1038,"children":1039},{},[1040],{"type":155,"value":977},{"type":150,"tag":417,"props":1042,"children":1043},{},[1044],{"type":155,"value":1045},"=",{"type":150,"tag":411,"props":1047,"children":1048},{},[1049],{"type":155,"value":415},{"type":150,"tag":417,"props":1051,"children":1052},{"separator":214},[1053],{"type":155,"value":421},{"type":150,"tag":411,"props":1055,"children":1056},{},[1057],{"type":155,"value":433},{"type":150,"tag":417,"props":1059,"children":1060},{"separator":214},[1061],{"type":155,"value":421},{"type":150,"tag":417,"props":1063,"children":1064},{},[1065],{"type":155,"value":442},{"type":150,"tag":217,"props":1067,"children":1068},{},[1069],{"type":155,"value":447},{"type":150,"tag":417,"props":1071,"children":1072},{"separator":214},[1073],{"type":155,"value":421},{"type":150,"tag":405,"props":1075,"children":1076},{},[1077],{"type":155,"value":926},{"type":150,"tag":223,"props":1079,"children":1080},{"encoding":225},[1081],{"type":155,"value":1082},"s = 1, 2, \\cdots, h",{"type":150,"tag":169,"props":1084,"children":1086},{"className":1085,"ariaHidden":214},[232],[1087,1114],{"type":150,"tag":169,"props":1088,"children":1090},{"className":1089},[237],[1091,1096,1101,1105,1110],{"type":150,"tag":169,"props":1092,"children":1095},{"className":1093,"style":1094},[242],"height:0.4306em;",[],{"type":150,"tag":169,"props":1097,"children":1099},{"className":1098},[248,489],[1100],{"type":155,"value":977},{"type":150,"tag":169,"props":1102,"children":1104},{"className":1103,"style":839},[558],[],{"type":150,"tag":169,"props":1106,"children":1108},{"className":1107},[844],[1109],{"type":155,"value":1045},{"type":150,"tag":169,"props":1111,"children":1113},{"className":1112,"style":839},[558],[],{"type":150,"tag":169,"props":1115,"children":1117},{"className":1116},[237],[1118,1123,1128,1133,1137,1142,1147,1151,1156,1160,1164,1169,1173],{"type":150,"tag":169,"props":1119,"children":1122},{"className":1120,"style":1121},[242],"height:0.8889em;vertical-align:-0.1944em;",[],{"type":150,"tag":169,"props":1124,"children":1126},{"className":1125},[248],[1127],{"type":155,"value":415},{"type":150,"tag":169,"props":1129,"children":1131},{"className":1130},[552],[1132],{"type":155,"value":421},{"type":150,"tag":169,"props":1134,"children":1136},{"className":1135,"style":559},[558],[],{"type":150,"tag":169,"props":1138,"children":1140},{"className":1139},[248],[1141],{"type":155,"value":433},{"type":150,"tag":169,"props":1143,"children":1145},{"className":1144},[552],[1146],{"type":155,"value":421},{"type":150,"tag":169,"props":1148,"children":1150},{"className":1149,"style":559},[558],[],{"type":150,"tag":169,"props":1152,"children":1154},{"className":1153},[630],[1155],{"type":155,"value":442},{"type":150,"tag":169,"props":1157,"children":1159},{"className":1158,"style":559},[558],[],{"type":150,"tag":169,"props":1161,"children":1163},{"className":1162,"style":559},[558],[],{"type":150,"tag":169,"props":1165,"children":1167},{"className":1166},[552],[1168],{"type":155,"value":421},{"type":150,"tag":169,"props":1170,"children":1172},{"className":1171,"style":559},[558],[],{"type":150,"tag":169,"props":1174,"children":1176},{"className":1175},[248,489],[1177],{"type":155,"value":926},{"type":155,"value":1179},"。那么：",{"type":150,"tag":169,"props":1181,"children":1183},{"className":1182},[172],[1184],{"type":150,"tag":169,"props":1185,"children":1187},{"className":1186},[177],[1188,1332],{"type":150,"tag":169,"props":1189,"children":1191},{"className":1190},[182],[1192],{"type":150,"tag":185,"props":1193,"children":1194},{"xmlns":187,"display":188},[1195],{"type":150,"tag":191,"props":1196,"children":1197},{},[1198,1327],{"type":150,"tag":195,"props":1199,"children":1200},{},[1201,1214,1218,1223,1250,1254,1280,1284,1288,1292,1296,1322],{"type":150,"tag":401,"props":1202,"children":1203},{},[1204,1209],{"type":150,"tag":405,"props":1205,"children":1206},{},[1207],{"type":155,"value":1208},"o",{"type":150,"tag":405,"props":1210,"children":1211},{},[1212],{"type":155,"value":1213},"t",{"type":150,"tag":417,"props":1215,"children":1216},{},[1217],{"type":155,"value":1045},{"type":150,"tag":417,"props":1219,"children":1220},{"stretchy":207},[1221],{"type":155,"value":1222},"[",{"type":150,"tag":1224,"props":1225,"children":1226},"msubsup",{},[1227,1231,1235],{"type":150,"tag":405,"props":1228,"children":1229},{},[1230],{"type":155,"value":1208},{"type":150,"tag":405,"props":1232,"children":1233},{},[1234],{"type":155,"value":1213},{"type":150,"tag":195,"props":1236,"children":1237},{},[1238,1242,1246],{"type":150,"tag":417,"props":1239,"children":1240},{"stretchy":207},[1241],{"type":155,"value":972},{"type":150,"tag":411,"props":1243,"children":1244},{},[1245],{"type":155,"value":415},{"type":150,"tag":417,"props":1247,"children":1248},{"stretchy":207},[1249],{"type":155,"value":982},{"type":150,"tag":417,"props":1251,"children":1252},{"separator":214},[1253],{"type":155,"value":421},{"type":150,"tag":1224,"props":1255,"children":1256},{},[1257,1261,1265],{"type":150,"tag":405,"props":1258,"children":1259},{},[1260],{"type":155,"value":1208},{"type":150,"tag":405,"props":1262,"children":1263},{},[1264],{"type":155,"value":1213},{"type":150,"tag":195,"props":1266,"children":1267},{},[1268,1272,1276],{"type":150,"tag":417,"props":1269,"children":1270},{"stretchy":207},[1271],{"type":155,"value":972},{"type":150,"tag":411,"props":1273,"children":1274},{},[1275],{"type":155,"value":433},{"type":150,"tag":417,"props":1277,"children":1278},{"stretchy":207},[1279],{"type":155,"value":982},{"type":150,"tag":417,"props":1281,"children":1282},{"separator":214},[1283],{"type":155,"value":421},{"type":150,"tag":417,"props":1285,"children":1286},{},[1287],{"type":155,"value":442},{"type":150,"tag":217,"props":1289,"children":1290},{},[1291],{"type":155,"value":447},{"type":150,"tag":417,"props":1293,"children":1294},{"separator":214},[1295],{"type":155,"value":421},{"type":150,"tag":1224,"props":1297,"children":1298},{},[1299,1303,1307],{"type":150,"tag":405,"props":1300,"children":1301},{},[1302],{"type":155,"value":1208},{"type":150,"tag":405,"props":1304,"children":1305},{},[1306],{"type":155,"value":1213},{"type":150,"tag":195,"props":1308,"children":1309},{},[1310,1314,1318],{"type":150,"tag":417,"props":1311,"children":1312},{"stretchy":207},[1313],{"type":155,"value":972},{"type":150,"tag":405,"props":1315,"children":1316},{},[1317],{"type":155,"value":926},{"type":150,"tag":417,"props":1319,"children":1320},{"stretchy":207},[1321],{"type":155,"value":982},{"type":150,"tag":417,"props":1323,"children":1324},{"stretchy":207},[1325],{"type":155,"value":1326},"]",{"type":150,"tag":223,"props":1328,"children":1329},{"encoding":225},[1330],{"type":155,"value":1331},"o_t = [o_t^{(1)}, o_t^{(2)}, \\cdots, o_t^{(h)}]",{"type":150,"tag":169,"props":1333,"children":1335},{"className":1334,"ariaHidden":214},[232],[1336,1416],{"type":150,"tag":169,"props":1337,"children":1339},{"className":1338},[237],[1340,1345,1403,1407,1412],{"type":150,"tag":169,"props":1341,"children":1344},{"className":1342,"style":1343},[242],"height:0.5806em;vertical-align:-0.15em;",[],{"type":150,"tag":169,"props":1346,"children":1348},{"className":1347},[248],[1349,1354],{"type":150,"tag":169,"props":1350,"children":1352},{"className":1351},[248,489],[1353],{"type":155,"value":1208},{"type":150,"tag":169,"props":1355,"children":1357},{"className":1356},[495],[1358],{"type":150,"tag":169,"props":1359,"children":1361},{"className":1360},[253,254],[1362,1392],{"type":150,"tag":169,"props":1363,"children":1365},{"className":1364},[259],[1366,1387],{"type":150,"tag":169,"props":1367,"children":1370},{"className":1368,"style":1369},[264],"height:0.2806em;",[1371],{"type":150,"tag":169,"props":1372,"children":1373},{"style":512},[1374,1378],{"type":150,"tag":169,"props":1375,"children":1377},{"className":1376,"style":517},[274],[],{"type":150,"tag":169,"props":1379,"children":1381},{"className":1380},[522,523,524,525],[1382],{"type":150,"tag":169,"props":1383,"children":1385},{"className":1384},[248,489,525],[1386],{"type":155,"value":1213},{"type":150,"tag":169,"props":1388,"children":1390},{"className":1389},[355],[1391],{"type":155,"value":358},{"type":150,"tag":169,"props":1393,"children":1395},{"className":1394},[259],[1396],{"type":150,"tag":169,"props":1397,"children":1399},{"className":1398,"style":544},[264],[1400],{"type":150,"tag":169,"props":1401,"children":1402},{},[],{"type":150,"tag":169,"props":1404,"children":1406},{"className":1405,"style":839},[558],[],{"type":150,"tag":169,"props":1408,"children":1410},{"className":1409},[844],[1411],{"type":155,"value":1045},{"type":150,"tag":169,"props":1413,"children":1415},{"className":1414,"style":839},[558],[],{"type":150,"tag":169,"props":1417,"children":1419},{"className":1418},[237],[1420,1425,1430,1521,1526,1530,1617,1622,1626,1631,1635,1639,1644,1648,1735],{"type":150,"tag":169,"props":1421,"children":1424},{"className":1422,"style":1423},[242],"height:1.2948em;vertical-align:-0.25em;",[],{"type":150,"tag":169,"props":1426,"children":1428},{"className":1427},[1004],[1429],{"type":155,"value":1222},{"type":150,"tag":169,"props":1431,"children":1433},{"className":1432},[248],[1434,1439],{"type":150,"tag":169,"props":1435,"children":1437},{"className":1436},[248,489],[1438],{"type":155,"value":1208},{"type":150,"tag":169,"props":1440,"children":1442},{"className":1441},[495],[1443],{"type":150,"tag":169,"props":1444,"children":1446},{"className":1445},[253,254],[1447,1509],{"type":150,"tag":169,"props":1448,"children":1450},{"className":1449},[259],[1451,1504],{"type":150,"tag":169,"props":1452,"children":1455},{"className":1453,"style":1454},[264],"height:1.0448em;",[1456,1473],{"type":150,"tag":169,"props":1457,"children":1459},{"style":1458},"top:-2.4542em;margin-left:0em;margin-right:0.05em;",[1460,1464],{"type":150,"tag":169,"props":1461,"children":1463},{"className":1462,"style":517},[274],[],{"type":150,"tag":169,"props":1465,"children":1467},{"className":1466},[522,523,524,525],[1468],{"type":150,"tag":169,"props":1469,"children":1471},{"className":1470},[248,489,525],[1472],{"type":155,"value":1213},{"type":150,"tag":169,"props":1474,"children":1476},{"style":1475},"top:-3.2198em;margin-right:0.05em;",[1477,1481],{"type":150,"tag":169,"props":1478,"children":1480},{"className":1479,"style":517},[274],[],{"type":150,"tag":169,"props":1482,"children":1484},{"className":1483},[522,523,524,525],[1485],{"type":150,"tag":169,"props":1486,"children":1488},{"className":1487},[248,525],[1489,1494,1499],{"type":150,"tag":169,"props":1490,"children":1492},{"className":1491},[1004,525],[1493],{"type":155,"value":972},{"type":150,"tag":169,"props":1495,"children":1497},{"className":1496},[248,525],[1498],{"type":155,"value":415},{"type":150,"tag":169,"props":1500,"children":1502},{"className":1501},[1015,525],[1503],{"type":155,"value":982},{"type":150,"tag":169,"props":1505,"children":1507},{"className":1506},[355],[1508],{"type":155,"value":358},{"type":150,"tag":169,"props":1510,"children":1512},{"className":1511},[259],[1513],{"type":150,"tag":169,"props":1514,"children":1517},{"className":1515,"style":1516},[264],"height:0.2458em;",[1518],{"type":150,"tag":169,"props":1519,"children":1520},{},[],{"type":150,"tag":169,"props":1522,"children":1524},{"className":1523},[552],[1525],{"type":155,"value":421},{"type":150,"tag":169,"props":1527,"children":1529},{"className":1528,"style":559},[558],[],{"type":150,"tag":169,"props":1531,"children":1533},{"className":1532},[248],[1534,1539],{"type":150,"tag":169,"props":1535,"children":1537},{"className":1536},[248,489],[1538],{"type":155,"value":1208},{"type":150,"tag":169,"props":1540,"children":1542},{"className":1541},[495],[1543],{"type":150,"tag":169,"props":1544,"children":1546},{"className":1545},[253,254],[1547,1606],{"type":150,"tag":169,"props":1548,"children":1550},{"className":1549},[259],[1551,1601],{"type":150,"tag":169,"props":1552,"children":1554},{"className":1553,"style":1454},[264],[1555,1571],{"type":150,"tag":169,"props":1556,"children":1557},{"style":1458},[1558,1562],{"type":150,"tag":169,"props":1559,"children":1561},{"className":1560,"style":517},[274],[],{"type":150,"tag":169,"props":1563,"children":1565},{"className":1564},[522,523,524,525],[1566],{"type":150,"tag":169,"props":1567,"children":1569},{"className":1568},[248,489,525],[1570],{"type":155,"value":1213},{"type":150,"tag":169,"props":1572,"children":1573},{"style":1475},[1574,1578],{"type":150,"tag":169,"props":1575,"children":1577},{"className":1576,"style":517},[274],[],{"type":150,"tag":169,"props":1579,"children":1581},{"className":1580},[522,523,524,525],[1582],{"type":150,"tag":169,"props":1583,"children":1585},{"className":1584},[248,525],[1586,1591,1596],{"type":150,"tag":169,"props":1587,"children":1589},{"className":1588},[1004,525],[1590],{"type":155,"value":972},{"type":150,"tag":169,"props":1592,"children":1594},{"className":1593},[248,525],[1595],{"type":155,"value":433},{"type":150,"tag":169,"props":1597,"children":1599},{"className":1598},[1015,525],[1600],{"type":155,"value":982},{"type":150,"tag":169,"props":1602,"children":1604},{"className":1603},[355],[1605],{"type":155,"value":358},{"type":150,"tag":169,"props":1607,"children":1609},{"className":1608},[259],[1610],{"type":150,"tag":169,"props":1611,"children":1613},{"className":1612,"style":1516},[264],[1614],{"type":150,"tag":169,"props":1615,"children":1616},{},[],{"type":150,"tag":169,"props":1618,"children":1620},{"className":1619},[552],[1621],{"type":155,"value":421},{"type":150,"tag":169,"props":1623,"children":1625},{"className":1624,"style":559},[558],[],{"type":150,"tag":169,"props":1627,"children":1629},{"className":1628},[630],[1630],{"type":155,"value":442},{"type":150,"tag":169,"props":1632,"children":1634},{"className":1633,"style":559},[558],[],{"type":150,"tag":169,"props":1636,"children":1638},{"className":1637,"style":559},[558],[],{"type":150,"tag":169,"props":1640,"children":1642},{"className":1641},[552],[1643],{"type":155,"value":421},{"type":150,"tag":169,"props":1645,"children":1647},{"className":1646,"style":559},[558],[],{"type":150,"tag":169,"props":1649,"children":1651},{"className":1650},[248],[1652,1657],{"type":150,"tag":169,"props":1653,"children":1655},{"className":1654},[248,489],[1656],{"type":155,"value":1208},{"type":150,"tag":169,"props":1658,"children":1660},{"className":1659},[495],[1661],{"type":150,"tag":169,"props":1662,"children":1664},{"className":1663},[253,254],[1665,1724],{"type":150,"tag":169,"props":1666,"children":1668},{"className":1667},[259],[1669,1719],{"type":150,"tag":169,"props":1670,"children":1672},{"className":1671,"style":1454},[264],[1673,1689],{"type":150,"tag":169,"props":1674,"children":1675},{"style":1458},[1676,1680],{"type":150,"tag":169,"props":1677,"children":1679},{"className":1678,"style":517},[274],[],{"type":150,"tag":169,"props":1681,"children":1683},{"className":1682},[522,523,524,525],[1684],{"type":150,"tag":169,"props":1685,"children":1687},{"className":1686},[248,489,525],[1688],{"type":155,"value":1213},{"type":150,"tag":169,"props":1690,"children":1691},{"style":1475},[1692,1696],{"type":150,"tag":169,"props":1693,"children":1695},{"className":1694,"style":517},[274],[],{"type":150,"tag":169,"props":1697,"children":1699},{"className":1698},[522,523,524,525],[1700],{"type":150,"tag":169,"props":1701,"children":1703},{"className":1702},[248,525],[1704,1709,1714],{"type":150,"tag":169,"props":1705,"children":1707},{"className":1706},[1004,525],[1708],{"type":155,"value":972},{"type":150,"tag":169,"props":1710,"children":1712},{"className":1711},[248,489,525],[1713],{"type":155,"value":926},{"type":150,"tag":169,"props":1715,"children":1717},{"className":1716},[1015,525],[1718],{"type":155,"value":982},{"type":150,"tag":169,"props":1720,"children":1722},{"className":1721},[355],[1723],{"type":155,"value":358},{"type":150,"tag":169,"props":1725,"children":1727},{"className":1726},[259],[1728],{"type":150,"tag":169,"props":1729,"children":1731},{"className":1730,"style":1516},[264],[1732],{"type":150,"tag":169,"props":1733,"children":1734},{},[],{"type":150,"tag":169,"props":1736,"children":1738},{"className":1737},[1015],[1739],{"type":155,"value":1326},{"type":150,"tag":169,"props":1741,"children":1743},{"className":1742},[172],[1744],{"type":150,"tag":169,"props":1745,"children":1747},{"className":1746},[177],[1748,2185],{"type":150,"tag":169,"props":1749,"children":1751},{"className":1750},[182],[1752],{"type":150,"tag":185,"props":1753,"children":1754},{"xmlns":187,"display":188},[1755],{"type":150,"tag":191,"props":1756,"children":1757},{},[1758,2180],{"type":150,"tag":195,"props":1759,"children":1760},{},[1761,1787,1791,1834,1838,1865,1869,1904,1908,1942,1946,1951],{"type":150,"tag":1224,"props":1762,"children":1763},{},[1764,1768,1772],{"type":150,"tag":405,"props":1765,"children":1766},{},[1767],{"type":155,"value":1208},{"type":150,"tag":405,"props":1769,"children":1770},{},[1771],{"type":155,"value":1213},{"type":150,"tag":195,"props":1773,"children":1774},{},[1775,1779,1783],{"type":150,"tag":417,"props":1776,"children":1777},{"stretchy":207},[1778],{"type":155,"value":972},{"type":150,"tag":405,"props":1780,"children":1781},{},[1782],{"type":155,"value":977},{"type":150,"tag":417,"props":1784,"children":1785},{"stretchy":207},[1786],{"type":155,"value":982},{"type":150,"tag":417,"props":1788,"children":1789},{},[1790],{"type":155,"value":1045},{"type":150,"tag":195,"props":1792,"children":1793},{},[1794,1800,1804,1808,1813,1818,1822,1826,1830],{"type":150,"tag":405,"props":1795,"children":1797},{"mathvariant":1796},"normal",[1798],{"type":155,"value":1799},"A",{"type":150,"tag":405,"props":1801,"children":1802},{"mathvariant":1796},[1803],{"type":155,"value":1213},{"type":150,"tag":405,"props":1805,"children":1806},{"mathvariant":1796},[1807],{"type":155,"value":1213},{"type":150,"tag":405,"props":1809,"children":1810},{"mathvariant":1796},[1811],{"type":155,"value":1812},"e",{"type":150,"tag":405,"props":1814,"children":1815},{"mathvariant":1796},[1816],{"type":155,"value":1817},"n",{"type":150,"tag":405,"props":1819,"children":1820},{"mathvariant":1796},[1821],{"type":155,"value":1213},{"type":150,"tag":405,"props":1823,"children":1824},{"mathvariant":1796},[1825],{"type":155,"value":739},{"type":150,"tag":405,"props":1827,"children":1828},{"mathvariant":1796},[1829],{"type":155,"value":1208},{"type":150,"tag":405,"props":1831,"children":1832},{"mathvariant":1796},[1833],{"type":155,"value":1817},{"type":150,"tag":417,"props":1835,"children":1836},{"stretchy":207},[1837],{"type":155,"value":972},{"type":150,"tag":1224,"props":1839,"children":1840},{},[1841,1846,1850],{"type":150,"tag":405,"props":1842,"children":1843},{},[1844],{"type":155,"value":1845},"q",{"type":150,"tag":405,"props":1847,"children":1848},{},[1849],{"type":155,"value":1213},{"type":150,"tag":195,"props":1851,"children":1852},{},[1853,1857,1861],{"type":150,"tag":417,"props":1854,"children":1855},{"stretchy":207},[1856],{"type":155,"value":972},{"type":150,"tag":405,"props":1858,"children":1859},{},[1860],{"type":155,"value":977},{"type":150,"tag":417,"props":1862,"children":1863},{"stretchy":207},[1864],{"type":155,"value":982},{"type":150,"tag":417,"props":1866,"children":1867},{"separator":214},[1868],{"type":155,"value":421},{"type":150,"tag":1224,"props":1870,"children":1871},{},[1872,1877,1889],{"type":150,"tag":405,"props":1873,"children":1874},{},[1875],{"type":155,"value":1876},"k",{"type":150,"tag":195,"props":1878,"children":1879},{},[1880,1885],{"type":150,"tag":417,"props":1881,"children":1882},{},[1883],{"type":155,"value":1884},"≤",{"type":150,"tag":405,"props":1886,"children":1887},{},[1888],{"type":155,"value":1213},{"type":150,"tag":195,"props":1890,"children":1891},{},[1892,1896,1900],{"type":150,"tag":417,"props":1893,"children":1894},{"stretchy":207},[1895],{"type":155,"value":972},{"type":150,"tag":405,"props":1897,"children":1898},{},[1899],{"type":155,"value":977},{"type":150,"tag":417,"props":1901,"children":1902},{"stretchy":207},[1903],{"type":155,"value":982},{"type":150,"tag":417,"props":1905,"children":1906},{"separator":214},[1907],{"type":155,"value":421},{"type":150,"tag":1224,"props":1909,"children":1910},{},[1911,1916,1927],{"type":150,"tag":405,"props":1912,"children":1913},{},[1914],{"type":155,"value":1915},"v",{"type":150,"tag":195,"props":1917,"children":1918},{},[1919,1923],{"type":150,"tag":417,"props":1920,"children":1921},{},[1922],{"type":155,"value":1884},{"type":150,"tag":405,"props":1924,"children":1925},{},[1926],{"type":155,"value":1213},{"type":150,"tag":195,"props":1928,"children":1929},{},[1930,1934,1938],{"type":150,"tag":417,"props":1931,"children":1932},{"stretchy":207},[1933],{"type":155,"value":972},{"type":150,"tag":405,"props":1935,"children":1936},{},[1937],{"type":155,"value":977},{"type":150,"tag":417,"props":1939,"children":1940},{"stretchy":207},[1941],{"type":155,"value":982},{"type":150,"tag":417,"props":1943,"children":1944},{"stretchy":207},[1945],{"type":155,"value":982},{"type":150,"tag":417,"props":1947,"children":1948},{},[1949],{"type":155,"value":1950},"≜",{"type":150,"tag":1952,"props":1953,"children":1954},"mfrac",{},[1955,2083],{"type":150,"tag":195,"props":1956,"children":1957},{},[1958,1982,1987,1992,1996,2022,2053,2057],{"type":150,"tag":1959,"props":1960,"children":1961},"munder",{},[1962,1967],{"type":150,"tag":417,"props":1963,"children":1964},{},[1965],{"type":155,"value":1966},"∑",{"type":150,"tag":195,"props":1968,"children":1969},{},[1970,1974,1978],{"type":150,"tag":405,"props":1971,"children":1972},{},[1973],{"type":155,"value":739},{"type":150,"tag":417,"props":1975,"children":1976},{},[1977],{"type":155,"value":1884},{"type":150,"tag":405,"props":1979,"children":1980},{},[1981],{"type":155,"value":1213},{"type":150,"tag":405,"props":1983,"children":1984},{},[1985],{"type":155,"value":1986},"exp",{"type":150,"tag":417,"props":1988,"children":1989},{},[1990],{"type":155,"value":1991},"⁡",{"type":150,"tag":417,"props":1993,"children":1994},{"stretchy":207},[1995],{"type":155,"value":972},{"type":150,"tag":1224,"props":1997,"children":1998},{},[1999,2003,2007],{"type":150,"tag":405,"props":2000,"children":2001},{},[2002],{"type":155,"value":1845},{"type":150,"tag":405,"props":2004,"children":2005},{},[2006],{"type":155,"value":1213},{"type":150,"tag":195,"props":2008,"children":2009},{},[2010,2014,2018],{"type":150,"tag":417,"props":2011,"children":2012},{"stretchy":207},[2013],{"type":155,"value":972},{"type":150,"tag":405,"props":2015,"children":2016},{},[2017],{"type":155,"value":977},{"type":150,"tag":417,"props":2019,"children":2020},{"stretchy":207},[2021],{"type":155,"value":982},{"type":150,"tag":1224,"props":2023,"children":2024},{},[2025,2029,2033],{"type":150,"tag":405,"props":2026,"children":2027},{},[2028],{"type":155,"value":1876},{"type":150,"tag":405,"props":2030,"children":2031},{},[2032],{"type":155,"value":739},{"type":150,"tag":195,"props":2034,"children":2035},{},[2036,2040,2044,2048],{"type":150,"tag":417,"props":2037,"children":2038},{"stretchy":207},[2039],{"type":155,"value":972},{"type":150,"tag":405,"props":2041,"children":2042},{},[2043],{"type":155,"value":977},{"type":150,"tag":417,"props":2045,"children":2046},{"stretchy":207},[2047],{"type":155,"value":982},{"type":150,"tag":405,"props":2049,"children":2050},{"mathvariant":1796},[2051],{"type":155,"value":2052},"⊤",{"type":150,"tag":417,"props":2054,"children":2055},{"stretchy":207},[2056],{"type":155,"value":982},{"type":150,"tag":1224,"props":2058,"children":2059},{},[2060,2064,2068],{"type":150,"tag":405,"props":2061,"children":2062},{},[2063],{"type":155,"value":1915},{"type":150,"tag":405,"props":2065,"children":2066},{},[2067],{"type":155,"value":739},{"type":150,"tag":195,"props":2069,"children":2070},{},[2071,2075,2079],{"type":150,"tag":417,"props":2072,"children":2073},{"stretchy":207},[2074],{"type":155,"value":972},{"type":150,"tag":405,"props":2076,"children":2077},{},[2078],{"type":155,"value":977},{"type":150,"tag":417,"props":2080,"children":2081},{"stretchy":207},[2082],{"type":155,"value":982},{"type":150,"tag":195,"props":2084,"children":2085},{},[2086,2108,2112,2116,2120,2146,2176],{"type":150,"tag":1959,"props":2087,"children":2088},{},[2089,2093],{"type":150,"tag":417,"props":2090,"children":2091},{},[2092],{"type":155,"value":1966},{"type":150,"tag":195,"props":2094,"children":2095},{},[2096,2100,2104],{"type":150,"tag":405,"props":2097,"children":2098},{},[2099],{"type":155,"value":739},{"type":150,"tag":417,"props":2101,"children":2102},{},[2103],{"type":155,"value":1884},{"type":150,"tag":405,"props":2105,"children":2106},{},[2107],{"type":155,"value":1213},{"type":150,"tag":405,"props":2109,"children":2110},{},[2111],{"type":155,"value":1986},{"type":150,"tag":417,"props":2113,"children":2114},{},[2115],{"type":155,"value":1991},{"type":150,"tag":417,"props":2117,"children":2118},{"stretchy":207},[2119],{"type":155,"value":972},{"type":150,"tag":1224,"props":2121,"children":2122},{},[2123,2127,2131],{"type":150,"tag":405,"props":2124,"children":2125},{},[2126],{"type":155,"value":1845},{"type":150,"tag":405,"props":2128,"children":2129},{},[2130],{"type":155,"value":1213},{"type":150,"tag":195,"props":2132,"children":2133},{},[2134,2138,2142],{"type":150,"tag":417,"props":2135,"children":2136},{"stretchy":207},[2137],{"type":155,"value":972},{"type":150,"tag":405,"props":2139,"children":2140},{},[2141],{"type":155,"value":977},{"type":150,"tag":417,"props":2143,"children":2144},{"stretchy":207},[2145],{"type":155,"value":982},{"type":150,"tag":1224,"props":2147,"children":2148},{},[2149,2153,2157],{"type":150,"tag":405,"props":2150,"children":2151},{},[2152],{"type":155,"value":1876},{"type":150,"tag":405,"props":2154,"children":2155},{},[2156],{"type":155,"value":739},{"type":150,"tag":195,"props":2158,"children":2159},{},[2160,2164,2168,2172],{"type":150,"tag":417,"props":2161,"children":2162},{"stretchy":207},[2163],{"type":155,"value":972},{"type":150,"tag":405,"props":2165,"children":2166},{},[2167],{"type":155,"value":977},{"type":150,"tag":417,"props":2169,"children":2170},{"stretchy":207},[2171],{"type":155,"value":982},{"type":150,"tag":405,"props":2173,"children":2174},{"mathvariant":1796},[2175],{"type":155,"value":2052},{"type":150,"tag":417,"props":2177,"children":2178},{"stretchy":207},[2179],{"type":155,"value":982},{"type":150,"tag":223,"props":2181,"children":2182},{"encoding":225},[2183],{"type":155,"value":2184},"o_t^{(s)} = \\mathrm{Attention}(q_t^{(s)}, k^{(s)}_{\\le t}, v^{(s)}_{\\le t}) \\triangleq \\frac{\\sum_{i \\le t} \\exp(q_t^{(s)} k_i^{(s)\\top}) v_i^{(s)}}{\\sum_{i \\le t} \\exp(q_t^{(s)} k_i^{(s)\\top})}",{"type":150,"tag":169,"props":2186,"children":2188},{"className":2187,"ariaHidden":214},[232],[2189,2298,2644],{"type":150,"tag":169,"props":2190,"children":2192},{"className":2191},[237],[2193,2198,2285,2289,2294],{"type":150,"tag":169,"props":2194,"children":2197},{"className":2195,"style":2196},[242],"height:1.2906em;vertical-align:-0.2458em;",[],{"type":150,"tag":169,"props":2199,"children":2201},{"className":2200},[248],[2202,2207],{"type":150,"tag":169,"props":2203,"children":2205},{"className":2204},[248,489],[2206],{"type":155,"value":1208},{"type":150,"tag":169,"props":2208,"children":2210},{"className":2209},[495],[2211],{"type":150,"tag":169,"props":2212,"children":2214},{"className":2213},[253,254],[2215,2274],{"type":150,"tag":169,"props":2216,"children":2218},{"className":2217},[259],[2219,2269],{"type":150,"tag":169,"props":2220,"children":2222},{"className":2221,"style":1454},[264],[2223,2239],{"type":150,"tag":169,"props":2224,"children":2225},{"style":1458},[2226,2230],{"type":150,"tag":169,"props":2227,"children":2229},{"className":2228,"style":517},[274],[],{"type":150,"tag":169,"props":2231,"children":2233},{"className":2232},[522,523,524,525],[2234],{"type":150,"tag":169,"props":2235,"children":2237},{"className":2236},[248,489,525],[2238],{"type":155,"value":1213},{"type":150,"tag":169,"props":2240,"children":2241},{"style":1475},[2242,2246],{"type":150,"tag":169,"props":2243,"children":2245},{"className":2244,"style":517},[274],[],{"type":150,"tag":169,"props":2247,"children":2249},{"className":2248},[522,523,524,525],[2250],{"type":150,"tag":169,"props":2251,"children":2253},{"className":2252},[248,525],[2254,2259,2264],{"type":150,"tag":169,"props":2255,"children":2257},{"className":2256},[1004,525],[2258],{"type":155,"value":972},{"type":150,"tag":169,"props":2260,"children":2262},{"className":2261},[248,489,525],[2263],{"type":155,"value":977},{"type":150,"tag":169,"props":2265,"children":2267},{"className":2266},[1015,525],[2268],{"type":155,"value":982},{"type":150,"tag":169,"props":2270,"children":2272},{"className":2271},[355],[2273],{"type":155,"value":358},{"type":150,"tag":169,"props":2275,"children":2277},{"className":2276},[259],[2278],{"type":150,"tag":169,"props":2279,"children":2281},{"className":2280,"style":1516},[264],[2282],{"type":150,"tag":169,"props":2283,"children":2284},{},[],{"type":150,"tag":169,"props":2286,"children":2288},{"className":2287,"style":839},[558],[],{"type":150,"tag":169,"props":2290,"children":2292},{"className":2291},[844],[2293],{"type":155,"value":1045},{"type":150,"tag":169,"props":2295,"children":2297},{"className":2296,"style":839},[558],[],{"type":150,"tag":169,"props":2299,"children":2301},{"className":2300},[237],[2302,2307,2317,2322,2411,2416,2420,2519,2524,2528,2625,2630,2634,2640],{"type":150,"tag":169,"props":2303,"children":2306},{"className":2304,"style":2305},[242],"height:1.4004em;vertical-align:-0.3556em;",[],{"type":150,"tag":169,"props":2308,"children":2310},{"className":2309},[248],[2311],{"type":150,"tag":169,"props":2312,"children":2315},{"className":2313},[248,2314],"mathrm",[2316],{"type":155,"value":12},{"type":150,"tag":169,"props":2318,"children":2320},{"className":2319},[1004],[2321],{"type":155,"value":972},{"type":150,"tag":169,"props":2323,"children":2325},{"className":2324},[248],[2326,2332],{"type":150,"tag":169,"props":2327,"children":2330},{"className":2328,"style":2329},[248,489],"margin-right:0.0359em;",[2331],{"type":155,"value":1845},{"type":150,"tag":169,"props":2333,"children":2335},{"className":2334},[495],[2336],{"type":150,"tag":169,"props":2337,"children":2339},{"className":2338},[253,254],[2340,2400],{"type":150,"tag":169,"props":2341,"children":2343},{"className":2342},[259],[2344,2395],{"type":150,"tag":169,"props":2345,"children":2347},{"className":2346,"style":1454},[264],[2348,2365],{"type":150,"tag":169,"props":2349,"children":2351},{"style":2350},"top:-2.4542em;margin-left:-0.0359em;margin-right:0.05em;",[2352,2356],{"type":150,"tag":169,"props":2353,"children":2355},{"className":2354,"style":517},[274],[],{"type":150,"tag":169,"props":2357,"children":2359},{"className":2358},[522,523,524,525],[2360],{"type":150,"tag":169,"props":2361,"children":2363},{"className":2362},[248,489,525],[2364],{"type":155,"value":1213},{"type":150,"tag":169,"props":2366,"children":2367},{"style":1475},[2368,2372],{"type":150,"tag":169,"props":2369,"children":2371},{"className":2370,"style":517},[274],[],{"type":150,"tag":169,"props":2373,"children":2375},{"className":2374},[522,523,524,525],[2376],{"type":150,"tag":169,"props":2377,"children":2379},{"className":2378},[248,525],[2380,2385,2390],{"type":150,"tag":169,"props":2381,"children":2383},{"className":2382},[1004,525],[2384],{"type":155,"value":972},{"type":150,"tag":169,"props":2386,"children":2388},{"className":2387},[248,489,525],[2389],{"type":155,"value":977},{"type":150,"tag":169,"props":2391,"children":2393},{"className":2392},[1015,525],[2394],{"type":155,"value":982},{"type":150,"tag":169,"props":2396,"children":2398},{"className":2397},[355],[2399],{"type":155,"value":358},{"type":150,"tag":169,"props":2401,"children":2403},{"className":2402},[259],[2404],{"type":150,"tag":169,"props":2405,"children":2407},{"className":2406,"style":1516},[264],[2408],{"type":150,"tag":169,"props":2409,"children":2410},{},[],{"type":150,"tag":169,"props":2412,"children":2414},{"className":2413},[552],[2415],{"type":155,"value":421},{"type":150,"tag":169,"props":2417,"children":2419},{"className":2418,"style":559},[558],[],{"type":150,"tag":169,"props":2421,"children":2423},{"className":2422},[248],[2424,2430],{"type":150,"tag":169,"props":2425,"children":2428},{"className":2426,"style":2427},[248,489],"margin-right:0.0315em;",[2429],{"type":155,"value":1876},{"type":150,"tag":169,"props":2431,"children":2433},{"className":2432},[495],[2434],{"type":150,"tag":169,"props":2435,"children":2437},{"className":2436},[253,254],[2438,2507],{"type":150,"tag":169,"props":2439,"children":2441},{"className":2440},[259],[2442,2502],{"type":150,"tag":169,"props":2443,"children":2445},{"className":2444,"style":1454},[264],[2446,2472],{"type":150,"tag":169,"props":2447,"children":2449},{"style":2448},"top:-2.4396em;margin-left:-0.0315em;margin-right:0.05em;",[2450,2454],{"type":150,"tag":169,"props":2451,"children":2453},{"className":2452,"style":517},[274],[],{"type":150,"tag":169,"props":2455,"children":2457},{"className":2456},[522,523,524,525],[2458],{"type":150,"tag":169,"props":2459,"children":2461},{"className":2460},[248,525],[2462,2467],{"type":150,"tag":169,"props":2463,"children":2465},{"className":2464},[844,525],[2466],{"type":155,"value":1884},{"type":150,"tag":169,"props":2468,"children":2470},{"className":2469},[248,489,525],[2471],{"type":155,"value":1213},{"type":150,"tag":169,"props":2473,"children":2474},{"style":1475},[2475,2479],{"type":150,"tag":169,"props":2476,"children":2478},{"className":2477,"style":517},[274],[],{"type":150,"tag":169,"props":2480,"children":2482},{"className":2481},[522,523,524,525],[2483],{"type":150,"tag":169,"props":2484,"children":2486},{"className":2485},[248,525],[2487,2492,2497],{"type":150,"tag":169,"props":2488,"children":2490},{"className":2489},[1004,525],[2491],{"type":155,"value":972},{"type":150,"tag":169,"props":2493,"children":2495},{"className":2494},[248,489,525],[2496],{"type":155,"value":977},{"type":150,"tag":169,"props":2498,"children":2500},{"className":2499},[1015,525],[2501],{"type":155,"value":982},{"type":150,"tag":169,"props":2503,"children":2505},{"className":2504},[355],[2506],{"type":155,"value":358},{"type":150,"tag":169,"props":2508,"children":2510},{"className":2509},[259],[2511],{"type":150,"tag":169,"props":2512,"children":2515},{"className":2513,"style":2514},[264],"height:0.3556em;",[2516],{"type":150,"tag":169,"props":2517,"children":2518},{},[],{"type":150,"tag":169,"props":2520,"children":2522},{"className":2521},[552],[2523],{"type":155,"value":421},{"type":150,"tag":169,"props":2525,"children":2527},{"className":2526,"style":559},[558],[],{"type":150,"tag":169,"props":2529,"children":2531},{"className":2530},[248],[2532,2537],{"type":150,"tag":169,"props":2533,"children":2535},{"className":2534,"style":2329},[248,489],[2536],{"type":155,"value":1915},{"type":150,"tag":169,"props":2538,"children":2540},{"className":2539},[495],[2541],{"type":150,"tag":169,"props":2542,"children":2544},{"className":2543},[253,254],[2545,2614],{"type":150,"tag":169,"props":2546,"children":2548},{"className":2547},[259],[2549,2609],{"type":150,"tag":169,"props":2550,"children":2552},{"className":2551,"style":1454},[264],[2553,2579],{"type":150,"tag":169,"props":2554,"children":2556},{"style":2555},"top:-2.4396em;margin-left:-0.0359em;margin-right:0.05em;",[2557,2561],{"type":150,"tag":169,"props":2558,"children":2560},{"className":2559,"style":517},[274],[],{"type":150,"tag":169,"props":2562,"children":2564},{"className":2563},[522,523,524,525],[2565],{"type":150,"tag":169,"props":2566,"children":2568},{"className":2567},[248,525],[2569,2574],{"type":150,"tag":169,"props":2570,"children":2572},{"className":2571},[844,525],[2573],{"type":155,"value":1884},{"type":150,"tag":169,"props":2575,"children":2577},{"className":2576},[248,489,525],[2578],{"type":155,"value":1213},{"type":150,"tag":169,"props":2580,"children":2581},{"style":1475},[2582,2586],{"type":150,"tag":169,"props":2583,"children":2585},{"className":2584,"style":517},[274],[],{"type":150,"tag":169,"props":2587,"children":2589},{"className":2588},[522,523,524,525],[2590],{"type":150,"tag":169,"props":2591,"children":2593},{"className":2592},[248,525],[2594,2599,2604],{"type":150,"tag":169,"props":2595,"children":2597},{"className":2596},[1004,525],[2598],{"type":155,"value":972},{"type":150,"tag":169,"props":2600,"children":2602},{"className":2601},[248,489,525],[2603],{"type":155,"value":977},{"type":150,"tag":169,"props":2605,"children":2607},{"className":2606},[1015,525],[2608],{"type":155,"value":982},{"type":150,"tag":169,"props":2610,"children":2612},{"className":2611},[355],[2613],{"type":155,"value":358},{"type":150,"tag":169,"props":2615,"children":2617},{"className":2616},[259],[2618],{"type":150,"tag":169,"props":2619,"children":2621},{"className":2620,"style":2514},[264],[2622],{"type":150,"tag":169,"props":2623,"children":2624},{},[],{"type":150,"tag":169,"props":2626,"children":2628},{"className":2627},[1015],[2629],{"type":155,"value":982},{"type":150,"tag":169,"props":2631,"children":2633},{"className":2632,"style":839},[558],[],{"type":150,"tag":169,"props":2635,"children":2638},{"className":2636},[844,2637],"amsrm",[2639],{"type":155,"value":1950},{"type":150,"tag":169,"props":2641,"children":2643},{"className":2642,"style":839},[558],[],{"type":150,"tag":169,"props":2645,"children":2647},{"className":2646},[237],[2648,2653],{"type":150,"tag":169,"props":2649,"children":2652},{"className":2650,"style":2651},[242],"height:3.1594em;vertical-align:-1.3297em;",[],{"type":150,"tag":169,"props":2654,"children":2656},{"className":2655},[248],[2657,2662,3370],{"type":150,"tag":169,"props":2658,"children":2661},{"className":2659},[1004,2660],"nulldelimiter",[],{"type":150,"tag":169,"props":2663,"children":2665},{"className":2664},[1952],[2666],{"type":150,"tag":169,"props":2667,"children":2669},{"className":2668},[253,254],[2670,3358],{"type":150,"tag":169,"props":2671,"children":2673},{"className":2672},[259],[2674,3353],{"type":150,"tag":169,"props":2675,"children":2678},{"className":2676,"style":2677},[264],"height:1.8297em;",[2679,2970,2984],{"type":150,"tag":169,"props":2680,"children":2682},{"style":2681},"top:-2.11em;",[2683,2688],{"type":150,"tag":169,"props":2684,"children":2687},{"className":2685,"style":2686},[274],"height:3.0448em;",[],{"type":150,"tag":169,"props":2689,"children":2691},{"className":2690},[248],[2692,2770,2774,2779,2784,2871,2965],{"type":150,"tag":169,"props":2693,"children":2696},{"className":2694},[2695],"mop",[2697,2705],{"type":150,"tag":169,"props":2698,"children":2703},{"className":2699,"style":2702},[2695,2700,2701],"op-symbol","small-op","position:relative;top:0em;",[2704],{"type":155,"value":1966},{"type":150,"tag":169,"props":2706,"children":2708},{"className":2707},[495],[2709],{"type":150,"tag":169,"props":2710,"children":2712},{"className":2711},[253,254],[2713,2758],{"type":150,"tag":169,"props":2714,"children":2716},{"className":2715},[259],[2717,2753],{"type":150,"tag":169,"props":2718,"children":2721},{"className":2719,"style":2720},[264],"height:0.162em;",[2722],{"type":150,"tag":169,"props":2723,"children":2725},{"style":2724},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[2726,2730],{"type":150,"tag":169,"props":2727,"children":2729},{"className":2728,"style":517},[274],[],{"type":150,"tag":169,"props":2731,"children":2733},{"className":2732},[522,523,524,525],[2734],{"type":150,"tag":169,"props":2735,"children":2737},{"className":2736},[248,525],[2738,2743,2748],{"type":150,"tag":169,"props":2739,"children":2741},{"className":2740},[248,489,525],[2742],{"type":155,"value":739},{"type":150,"tag":169,"props":2744,"children":2746},{"className":2745},[844,525],[2747],{"type":155,"value":1884},{"type":150,"tag":169,"props":2749,"children":2751},{"className":2750},[248,489,525],[2752],{"type":155,"value":1213},{"type":150,"tag":169,"props":2754,"children":2756},{"className":2755},[355],[2757],{"type":155,"value":358},{"type":150,"tag":169,"props":2759,"children":2761},{"className":2760},[259],[2762],{"type":150,"tag":169,"props":2763,"children":2766},{"className":2764,"style":2765},[264],"height:0.3949em;",[2767],{"type":150,"tag":169,"props":2768,"children":2769},{},[],{"type":150,"tag":169,"props":2771,"children":2773},{"className":2772,"style":559},[558],[],{"type":150,"tag":169,"props":2775,"children":2777},{"className":2776},[2695],[2778],{"type":155,"value":1986},{"type":150,"tag":169,"props":2780,"children":2782},{"className":2781},[1004],[2783],{"type":155,"value":972},{"type":150,"tag":169,"props":2785,"children":2787},{"className":2786},[248],[2788,2793],{"type":150,"tag":169,"props":2789,"children":2791},{"className":2790,"style":2329},[248,489],[2792],{"type":155,"value":1845},{"type":150,"tag":169,"props":2794,"children":2796},{"className":2795},[495],[2797],{"type":150,"tag":169,"props":2798,"children":2800},{"className":2799},[253,254],[2801,2860],{"type":150,"tag":169,"props":2802,"children":2804},{"className":2803},[259],[2805,2855],{"type":150,"tag":169,"props":2806,"children":2808},{"className":2807,"style":1454},[264],[2809,2825],{"type":150,"tag":169,"props":2810,"children":2811},{"style":2350},[2812,2816],{"type":150,"tag":169,"props":2813,"children":2815},{"className":2814,"style":517},[274],[],{"type":150,"tag":169,"props":2817,"children":2819},{"className":2818},[522,523,524,525],[2820],{"type":150,"tag":169,"props":2821,"children":2823},{"className":2822},[248,489,525],[2824],{"type":155,"value":1213},{"type":150,"tag":169,"props":2826,"children":2827},{"style":1475},[2828,2832],{"type":150,"tag":169,"props":2829,"children":2831},{"className":2830,"style":517},[274],[],{"type":150,"tag":169,"props":2833,"children":2835},{"className":2834},[522,523,524,525],[2836],{"type":150,"tag":169,"props":2837,"children":2839},{"className":2838},[248,525],[2840,2845,2850],{"type":150,"tag":169,"props":2841,"children":2843},{"className":2842},[1004,525],[2844],{"type":155,"value":972},{"type":150,"tag":169,"props":2846,"children":2848},{"className":2847},[248,489,525],[2849],{"type":155,"value":977},{"type":150,"tag":169,"props":2851,"children":2853},{"className":2852},[1015,525],[2854],{"type":155,"value":982},{"type":150,"tag":169,"props":2856,"children":2858},{"className":2857},[355],[2859],{"type":155,"value":358},{"type":150,"tag":169,"props":2861,"children":2863},{"className":2862},[259],[2864],{"type":150,"tag":169,"props":2865,"children":2867},{"className":2866,"style":1516},[264],[2868],{"type":150,"tag":169,"props":2869,"children":2870},{},[],{"type":150,"tag":169,"props":2872,"children":2874},{"className":2873},[248],[2875,2880],{"type":150,"tag":169,"props":2876,"children":2878},{"className":2877,"style":2427},[248,489],[2879],{"type":155,"value":1876},{"type":150,"tag":169,"props":2881,"children":2883},{"className":2882},[495],[2884],{"type":150,"tag":169,"props":2885,"children":2887},{"className":2886},[253,254],[2888,2953],{"type":150,"tag":169,"props":2889,"children":2891},{"className":2890},[259],[2892,2948],{"type":150,"tag":169,"props":2893,"children":2895},{"className":2894,"style":1454},[264],[2896,2913],{"type":150,"tag":169,"props":2897,"children":2899},{"style":2898},"top:-2.4231em;margin-left:-0.0315em;margin-right:0.05em;",[2900,2904],{"type":150,"tag":169,"props":2901,"children":2903},{"className":2902,"style":517},[274],[],{"type":150,"tag":169,"props":2905,"children":2907},{"className":2906},[522,523,524,525],[2908],{"type":150,"tag":169,"props":2909,"children":2911},{"className":2910},[248,489,525],[2912],{"type":155,"value":739},{"type":150,"tag":169,"props":2914,"children":2915},{"style":1475},[2916,2920],{"type":150,"tag":169,"props":2917,"children":2919},{"className":2918,"style":517},[274],[],{"type":150,"tag":169,"props":2921,"children":2923},{"className":2922},[522,523,524,525],[2924],{"type":150,"tag":169,"props":2925,"children":2927},{"className":2926},[248,525],[2928,2933,2938,2943],{"type":150,"tag":169,"props":2929,"children":2931},{"className":2930},[1004,525],[2932],{"type":155,"value":972},{"type":150,"tag":169,"props":2934,"children":2936},{"className":2935},[248,489,525],[2937],{"type":155,"value":977},{"type":150,"tag":169,"props":2939,"children":2941},{"className":2940},[1015,525],[2942],{"type":155,"value":982},{"type":150,"tag":169,"props":2944,"children":2946},{"className":2945},[248,525],[2947],{"type":155,"value":2052},{"type":150,"tag":169,"props":2949,"children":2951},{"className":2950},[355],[2952],{"type":155,"value":358},{"type":150,"tag":169,"props":2954,"children":2956},{"className":2955},[259],[2957],{"type":150,"tag":169,"props":2958,"children":2961},{"className":2959,"style":2960},[264],"height:0.2769em;",[2962],{"type":150,"tag":169,"props":2963,"children":2964},{},[],{"type":150,"tag":169,"props":2966,"children":2968},{"className":2967},[1015],[2969],{"type":155,"value":982},{"type":150,"tag":169,"props":2971,"children":2973},{"style":2972},"top:-3.2748em;",[2974,2978],{"type":150,"tag":169,"props":2975,"children":2977},{"className":2976,"style":2686},[274],[],{"type":150,"tag":169,"props":2979,"children":2983},{"className":2980,"style":2982},[2981],"frac-line","border-bottom-width:0.04em;",[],{"type":150,"tag":169,"props":2985,"children":2987},{"style":2986},"top:-3.8297em;",[2988,2992],{"type":150,"tag":169,"props":2989,"children":2991},{"className":2990,"style":2686},[274],[],{"type":150,"tag":169,"props":2993,"children":2995},{"className":2994},[248],[2996,3067,3071,3076,3081,3168,3260,3265],{"type":150,"tag":169,"props":2997,"children":2999},{"className":2998},[2695],[3000,3005],{"type":150,"tag":169,"props":3001,"children":3003},{"className":3002,"style":2702},[2695,2700,2701],[3004],{"type":155,"value":1966},{"type":150,"tag":169,"props":3006,"children":3008},{"className":3007},[495],[3009],{"type":150,"tag":169,"props":3010,"children":3012},{"className":3011},[253,254],[3013,3056],{"type":150,"tag":169,"props":3014,"children":3016},{"className":3015},[259],[3017,3051],{"type":150,"tag":169,"props":3018,"children":3020},{"className":3019,"style":2720},[264],[3021],{"type":150,"tag":169,"props":3022,"children":3023},{"style":2724},[3024,3028],{"type":150,"tag":169,"props":3025,"children":3027},{"className":3026,"style":517},[274],[],{"type":150,"tag":169,"props":3029,"children":3031},{"className":3030},[522,523,524,525],[3032],{"type":150,"tag":169,"props":3033,"children":3035},{"className":3034},[248,525],[3036,3041,3046],{"type":150,"tag":169,"props":3037,"children":3039},{"className":3038},[248,489,525],[3040],{"type":155,"value":739},{"type":150,"tag":169,"props":3042,"children":3044},{"className":3043},[844,525],[3045],{"type":155,"value":1884},{"type":150,"tag":169,"props":3047,"children":3049},{"className":3048},[248,489,525],[3050],{"type":155,"value":1213},{"type":150,"tag":169,"props":3052,"children":3054},{"className":3053},[355],[3055],{"type":155,"value":358},{"type":150,"tag":169,"props":3057,"children":3059},{"className":3058},[259],[3060],{"type":150,"tag":169,"props":3061,"children":3063},{"className":3062,"style":2765},[264],[3064],{"type":150,"tag":169,"props":3065,"children":3066},{},[],{"type":150,"tag":169,"props":3068,"children":3070},{"className":3069,"style":559},[558],[],{"type":150,"tag":169,"props":3072,"children":3074},{"className":3073},[2695],[3075],{"type":155,"value":1986},{"type":150,"tag":169,"props":3077,"children":3079},{"className":3078},[1004],[3080],{"type":155,"value":972},{"type":150,"tag":169,"props":3082,"children":3084},{"className":3083},[248],[3085,3090],{"type":150,"tag":169,"props":3086,"children":3088},{"className":3087,"style":2329},[248,489],[3089],{"type":155,"value":1845},{"type":150,"tag":169,"props":3091,"children":3093},{"className":3092},[495],[3094],{"type":150,"tag":169,"props":3095,"children":3097},{"className":3096},[253,254],[3098,3157],{"type":150,"tag":169,"props":3099,"children":3101},{"className":3100},[259],[3102,3152],{"type":150,"tag":169,"props":3103,"children":3105},{"className":3104,"style":1454},[264],[3106,3122],{"type":150,"tag":169,"props":3107,"children":3108},{"style":2350},[3109,3113],{"type":150,"tag":169,"props":3110,"children":3112},{"className":3111,"style":517},[274],[],{"type":150,"tag":169,"props":3114,"children":3116},{"className":3115},[522,523,524,525],[3117],{"type":150,"tag":169,"props":3118,"children":3120},{"className":3119},[248,489,525],[3121],{"type":155,"value":1213},{"type":150,"tag":169,"props":3123,"children":3124},{"style":1475},[3125,3129],{"type":150,"tag":169,"props":3126,"children":3128},{"className":3127,"style":517},[274],[],{"type":150,"tag":169,"props":3130,"children":3132},{"className":3131},[522,523,524,525],[3133],{"type":150,"tag":169,"props":3134,"children":3136},{"className":3135},[248,525],[3137,3142,3147],{"type":150,"tag":169,"props":3138,"children":3140},{"className":3139},[1004,525],[3141],{"type":155,"value":972},{"type":150,"tag":169,"props":3143,"children":3145},{"className":3144},[248,489,525],[3146],{"type":155,"value":977},{"type":150,"tag":169,"props":3148,"children":3150},{"className":3149},[1015,525],[3151],{"type":155,"value":982},{"type":150,"tag":169,"props":3153,"children":3155},{"className":3154},[355],[3156],{"type":155,"value":358},{"type":150,"tag":169,"props":3158,"children":3160},{"className":3159},[259],[3161],{"type":150,"tag":169,"props":3162,"children":3164},{"className":3163,"style":1516},[264],[3165],{"type":150,"tag":169,"props":3166,"children":3167},{},[],{"type":150,"tag":169,"props":3169,"children":3171},{"className":3170},[248],[3172,3177],{"type":150,"tag":169,"props":3173,"children":3175},{"className":3174,"style":2427},[248,489],[3176],{"type":155,"value":1876},{"type":150,"tag":169,"props":3178,"children":3180},{"className":3179},[495],[3181],{"type":150,"tag":169,"props":3182,"children":3184},{"className":3183},[253,254],[3185,3249],{"type":150,"tag":169,"props":3186,"children":3188},{"className":3187},[259],[3189,3244],{"type":150,"tag":169,"props":3190,"children":3192},{"className":3191,"style":1454},[264],[3193,3209],{"type":150,"tag":169,"props":3194,"children":3195},{"style":2898},[3196,3200],{"type":150,"tag":169,"props":3197,"children":3199},{"className":3198,"style":517},[274],[],{"type":150,"tag":169,"props":3201,"children":3203},{"className":3202},[522,523,524,525],[3204],{"type":150,"tag":169,"props":3205,"children":3207},{"className":3206},[248,489,525],[3208],{"type":155,"value":739},{"type":150,"tag":169,"props":3210,"children":3211},{"style":1475},[3212,3216],{"type":150,"tag":169,"props":3213,"children":3215},{"className":3214,"style":517},[274],[],{"type":150,"tag":169,"props":3217,"children":3219},{"className":3218},[522,523,524,525],[3220],{"type":150,"tag":169,"props":3221,"children":3223},{"className":3222},[248,525],[3224,3229,3234,3239],{"type":150,"tag":169,"props":3225,"children":3227},{"className":3226},[1004,525],[3228],{"type":155,"value":972},{"type":150,"tag":169,"props":3230,"children":3232},{"className":3231},[248,489,525],[3233],{"type":155,"value":977},{"type":150,"tag":169,"props":3235,"children":3237},{"className":3236},[1015,525],[3238],{"type":155,"value":982},{"type":150,"tag":169,"props":3240,"children":3242},{"className":3241},[248,525],[3243],{"type":155,"value":2052},{"type":150,"tag":169,"props":3245,"children":3247},{"className":3246},[355],[3248],{"type":155,"value":358},{"type":150,"tag":169,"props":3250,"children":3252},{"className":3251},[259],[3253],{"type":150,"tag":169,"props":3254,"children":3256},{"className":3255,"style":2960},[264],[3257],{"type":150,"tag":169,"props":3258,"children":3259},{},[],{"type":150,"tag":169,"props":3261,"children":3263},{"className":3262},[1015],[3264],{"type":155,"value":982},{"type":150,"tag":169,"props":3266,"children":3268},{"className":3267},[248],[3269,3274],{"type":150,"tag":169,"props":3270,"children":3272},{"className":3271,"style":2329},[248,489],[3273],{"type":155,"value":1915},{"type":150,"tag":169,"props":3275,"children":3277},{"className":3276},[495],[3278],{"type":150,"tag":169,"props":3279,"children":3281},{"className":3280},[253,254],[3282,3342],{"type":150,"tag":169,"props":3283,"children":3285},{"className":3284},[259],[3286,3337],{"type":150,"tag":169,"props":3287,"children":3289},{"className":3288,"style":1454},[264],[3290,3307],{"type":150,"tag":169,"props":3291,"children":3293},{"style":3292},"top:-2.4231em;margin-left:-0.0359em;margin-right:0.05em;",[3294,3298],{"type":150,"tag":169,"props":3295,"children":3297},{"className":3296,"style":517},[274],[],{"type":150,"tag":169,"props":3299,"children":3301},{"className":3300},[522,523,524,525],[3302],{"type":150,"tag":169,"props":3303,"children":3305},{"className":3304},[248,489,525],[3306],{"type":155,"value":739},{"type":150,"tag":169,"props":3308,"children":3309},{"style":1475},[3310,3314],{"type":150,"tag":169,"props":3311,"children":3313},{"className":3312,"style":517},[274],[],{"type":150,"tag":169,"props":3315,"children":3317},{"className":3316},[522,523,524,525],[3318],{"type":150,"tag":169,"props":3319,"children":3321},{"className":3320},[248,525],[3322,3327,3332],{"type":150,"tag":169,"props":3323,"children":3325},{"className":3324},[1004,525],[3326],{"type":155,"value":972},{"type":150,"tag":169,"props":3328,"children":3330},{"className":3329},[248,489,525],[3331],{"type":155,"value":977},{"type":150,"tag":169,"props":3333,"children":3335},{"className":3334},[1015,525],[3336],{"type":155,"value":982},{"type":150,"tag":169,"props":3338,"children":3340},{"className":3339},[355],[3341],{"type":155,"value":358},{"type":150,"tag":169,"props":3343,"children":3345},{"className":3344},[259],[3346],{"type":150,"tag":169,"props":3347,"children":3349},{"className":3348,"style":2960},[264],[3350],{"type":150,"tag":169,"props":3351,"children":3352},{},[],{"type":150,"tag":169,"props":3354,"children":3356},{"className":3355},[355],[3357],{"type":155,"value":358},{"type":150,"tag":169,"props":3359,"children":3361},{"className":3360},[259],[3362],{"type":150,"tag":169,"props":3363,"children":3366},{"className":3364,"style":3365},[264],"height:1.3297em;",[3367],{"type":150,"tag":169,"props":3368,"children":3369},{},[],{"type":150,"tag":169,"props":3371,"children":3373},{"className":3372},[1015,2660],[],{"type":150,"tag":169,"props":3375,"children":3377},{"className":3376},[172],[3378],{"type":150,"tag":169,"props":3379,"children":3381},{"className":3380},[177],[3382,3683],{"type":150,"tag":169,"props":3383,"children":3385},{"className":3384},[182],[3386],{"type":150,"tag":185,"props":3387,"children":3388},{"xmlns":187,"display":188},[3389],{"type":150,"tag":191,"props":3390,"children":3391},{},[3392,3678],{"type":150,"tag":195,"props":3393,"children":3394},{},[3395,3421,3425,3436,3463,3467,3485,3489,3493,3519,3523,3534,3560,3564,3582,3586,3589,3615,3619,3630,3656,3660],{"type":150,"tag":1224,"props":3396,"children":3397},{},[3398,3402,3406],{"type":150,"tag":405,"props":3399,"children":3400},{},[3401],{"type":155,"value":1845},{"type":150,"tag":405,"props":3403,"children":3404},{},[3405],{"type":155,"value":739},{"type":150,"tag":195,"props":3407,"children":3408},{},[3409,3413,3417],{"type":150,"tag":417,"props":3410,"children":3411},{"stretchy":207},[3412],{"type":155,"value":972},{"type":150,"tag":405,"props":3414,"children":3415},{},[3416],{"type":155,"value":977},{"type":150,"tag":417,"props":3418,"children":3419},{"stretchy":207},[3420],{"type":155,"value":982},{"type":150,"tag":417,"props":3422,"children":3423},{},[3424],{"type":155,"value":1045},{"type":150,"tag":401,"props":3426,"children":3427},{},[3428,3432],{"type":150,"tag":405,"props":3429,"children":3430},{},[3431],{"type":155,"value":409},{"type":150,"tag":405,"props":3433,"children":3434},{},[3435],{"type":155,"value":739},{"type":150,"tag":1224,"props":3437,"children":3438},{},[3439,3444,3448],{"type":150,"tag":405,"props":3440,"children":3441},{},[3442],{"type":155,"value":3443},"W",{"type":150,"tag":405,"props":3445,"children":3446},{},[3447],{"type":155,"value":1845},{"type":150,"tag":195,"props":3449,"children":3450},{},[3451,3455,3459],{"type":150,"tag":417,"props":3452,"children":3453},{"stretchy":207},[3454],{"type":155,"value":972},{"type":150,"tag":405,"props":3456,"children":3457},{},[3458],{"type":155,"value":977},{"type":150,"tag":417,"props":3460,"children":3461},{"stretchy":207},[3462],{"type":155,"value":982},{"type":150,"tag":417,"props":3464,"children":3465},{},[3466],{"type":155,"value":744},{"type":150,"tag":746,"props":3468,"children":3469},{},[3470,3474],{"type":150,"tag":405,"props":3471,"children":3472},{"mathvariant":751},[3473],{"type":155,"value":754},{"type":150,"tag":401,"props":3475,"children":3476},{},[3477,3481],{"type":150,"tag":405,"props":3478,"children":3479},{},[3480],{"type":155,"value":759},{"type":150,"tag":405,"props":3482,"children":3483},{},[3484],{"type":155,"value":1876},{"type":150,"tag":417,"props":3486,"children":3487},{"separator":214},[3488],{"type":155,"value":421},{"type":150,"tag":558,"props":3490,"children":3492},{"width":3491},"1em",[],{"type":150,"tag":1224,"props":3494,"children":3495},{},[3496,3500,3504],{"type":150,"tag":405,"props":3497,"children":3498},{},[3499],{"type":155,"value":1876},{"type":150,"tag":405,"props":3501,"children":3502},{},[3503],{"type":155,"value":739},{"type":150,"tag":195,"props":3505,"children":3506},{},[3507,3511,3515],{"type":150,"tag":417,"props":3508,"children":3509},{"stretchy":207},[3510],{"type":155,"value":972},{"type":150,"tag":405,"props":3512,"children":3513},{},[3514],{"type":155,"value":977},{"type":150,"tag":417,"props":3516,"children":3517},{"stretchy":207},[3518],{"type":155,"value":982},{"type":150,"tag":417,"props":3520,"children":3521},{},[3522],{"type":155,"value":1045},{"type":150,"tag":401,"props":3524,"children":3525},{},[3526,3530],{"type":150,"tag":405,"props":3527,"children":3528},{},[3529],{"type":155,"value":409},{"type":150,"tag":405,"props":3531,"children":3532},{},[3533],{"type":155,"value":739},{"type":150,"tag":1224,"props":3535,"children":3536},{},[3537,3541,3545],{"type":150,"tag":405,"props":3538,"children":3539},{},[3540],{"type":155,"value":3443},{"type":150,"tag":405,"props":3542,"children":3543},{},[3544],{"type":155,"value":1876},{"type":150,"tag":195,"props":3546,"children":3547},{},[3548,3552,3556],{"type":150,"tag":417,"props":3549,"children":3550},{"stretchy":207},[3551],{"type":155,"value":972},{"type":150,"tag":405,"props":3553,"children":3554},{},[3555],{"type":155,"value":977},{"type":150,"tag":417,"props":3557,"children":3558},{"stretchy":207},[3559],{"type":155,"value":982},{"type":150,"tag":417,"props":3561,"children":3562},{},[3563],{"type":155,"value":744},{"type":150,"tag":746,"props":3565,"children":3566},{},[3567,3571],{"type":150,"tag":405,"props":3568,"children":3569},{"mathvariant":751},[3570],{"type":155,"value":754},{"type":150,"tag":401,"props":3572,"children":3573},{},[3574,3578],{"type":150,"tag":405,"props":3575,"children":3576},{},[3577],{"type":155,"value":759},{"type":150,"tag":405,"props":3579,"children":3580},{},[3581],{"type":155,"value":1876},{"type":150,"tag":417,"props":3583,"children":3584},{"separator":214},[3585],{"type":155,"value":421},{"type":150,"tag":558,"props":3587,"children":3588},{"width":3491},[],{"type":150,"tag":1224,"props":3590,"children":3591},{},[3592,3596,3600],{"type":150,"tag":405,"props":3593,"children":3594},{},[3595],{"type":155,"value":1915},{"type":150,"tag":405,"props":3597,"children":3598},{},[3599],{"type":155,"value":739},{"type":150,"tag":195,"props":3601,"children":3602},{},[3603,3607,3611],{"type":150,"tag":417,"props":3604,"children":3605},{"stretchy":207},[3606],{"type":155,"value":972},{"type":150,"tag":405,"props":3608,"children":3609},{},[3610],{"type":155,"value":977},{"type":150,"tag":417,"props":3612,"children":3613},{"stretchy":207},[3614],{"type":155,"value":982},{"type":150,"tag":417,"props":3616,"children":3617},{},[3618],{"type":155,"value":1045},{"type":150,"tag":401,"props":3620,"children":3621},{},[3622,3626],{"type":150,"tag":405,"props":3623,"children":3624},{},[3625],{"type":155,"value":409},{"type":150,"tag":405,"props":3627,"children":3628},{},[3629],{"type":155,"value":739},{"type":150,"tag":1224,"props":3631,"children":3632},{},[3633,3637,3641],{"type":150,"tag":405,"props":3634,"children":3635},{},[3636],{"type":155,"value":3443},{"type":150,"tag":405,"props":3638,"children":3639},{},[3640],{"type":155,"value":1915},{"type":150,"tag":195,"props":3642,"children":3643},{},[3644,3648,3652],{"type":150,"tag":417,"props":3645,"children":3646},{"stretchy":207},[3647],{"type":155,"value":972},{"type":150,"tag":405,"props":3649,"children":3650},{},[3651],{"type":155,"value":977},{"type":150,"tag":417,"props":3653,"children":3654},{"stretchy":207},[3655],{"type":155,"value":982},{"type":150,"tag":417,"props":3657,"children":3658},{},[3659],{"type":155,"value":744},{"type":150,"tag":746,"props":3661,"children":3662},{},[3663,3667],{"type":150,"tag":405,"props":3664,"children":3665},{"mathvariant":751},[3666],{"type":155,"value":754},{"type":150,"tag":401,"props":3668,"children":3669},{},[3670,3674],{"type":150,"tag":405,"props":3671,"children":3672},{},[3673],{"type":155,"value":759},{"type":150,"tag":405,"props":3675,"children":3676},{},[3677],{"type":155,"value":1915},{"type":150,"tag":223,"props":3679,"children":3680},{"encoding":225},[3681],{"type":155,"value":3682},"q_i^{(s)} = x_i W_q^{(s)} \\in \\mathbb{R}^{d_k}, \\quad k_i^{(s)} = x_i W_k^{(s)} \\in \\mathbb{R}^{d_k}, \\quad v_i^{(s)} = x_i W_v^{(s)} \\in \\mathbb{R}^{d_v}",{"type":150,"tag":169,"props":3684,"children":3686},{"className":3685,"ariaHidden":214},[232],[3687,3796,3967,4193,4361,4579,4746],{"type":150,"tag":169,"props":3688,"children":3690},{"className":3689},[237],[3691,3696,3783,3787,3792],{"type":150,"tag":169,"props":3692,"children":3695},{"className":3693,"style":3694},[242],"height:1.3217em;vertical-align:-0.2769em;",[],{"type":150,"tag":169,"props":3697,"children":3699},{"className":3698},[248],[3700,3705],{"type":150,"tag":169,"props":3701,"children":3703},{"className":3702,"style":2329},[248,489],[3704],{"type":155,"value":1845},{"type":150,"tag":169,"props":3706,"children":3708},{"className":3707},[495],[3709],{"type":150,"tag":169,"props":3710,"children":3712},{"className":3711},[253,254],[3713,3772],{"type":150,"tag":169,"props":3714,"children":3716},{"className":3715},[259],[3717,3767],{"type":150,"tag":169,"props":3718,"children":3720},{"className":3719,"style":1454},[264],[3721,3737],{"type":150,"tag":169,"props":3722,"children":3723},{"style":3292},[3724,3728],{"type":150,"tag":169,"props":3725,"children":3727},{"className":3726,"style":517},[274],[],{"type":150,"tag":169,"props":3729,"children":3731},{"className":3730},[522,523,524,525],[3732],{"type":150,"tag":169,"props":3733,"children":3735},{"className":3734},[248,489,525],[3736],{"type":155,"value":739},{"type":150,"tag":169,"props":3738,"children":3739},{"style":1475},[3740,3744],{"type":150,"tag":169,"props":3741,"children":3743},{"className":3742,"style":517},[274],[],{"type":150,"tag":169,"props":3745,"children":3747},{"className":3746},[522,523,524,525],[3748],{"type":150,"tag":169,"props":3749,"children":3751},{"className":3750},[248,525],[3752,3757,3762],{"type":150,"tag":169,"props":3753,"children":3755},{"className":3754},[1004,525],[3756],{"type":155,"value":972},{"type":150,"tag":169,"props":3758,"children":3760},{"className":3759},[248,489,525],[3761],{"type":155,"value":977},{"type":150,"tag":169,"props":3763,"children":3765},{"className":3764},[1015,525],[3766],{"type":155,"value":982},{"type":150,"tag":169,"props":3768,"children":3770},{"className":3769},[355],[3771],{"type":155,"value":358},{"type":150,"tag":169,"props":3773,"children":3775},{"className":3774},[259],[3776],{"type":150,"tag":169,"props":3777,"children":3779},{"className":3778,"style":2960},[264],[3780],{"type":150,"tag":169,"props":3781,"children":3782},{},[],{"type":150,"tag":169,"props":3784,"children":3786},{"className":3785,"style":839},[558],[],{"type":150,"tag":169,"props":3788,"children":3790},{"className":3789},[844],[3791],{"type":155,"value":1045},{"type":150,"tag":169,"props":3793,"children":3795},{"className":3794,"style":839},[558],[],{"type":150,"tag":169,"props":3797,"children":3799},{"className":3798},[237],[3800,3805,3862,3954,3958,3963],{"type":150,"tag":169,"props":3801,"children":3804},{"className":3802,"style":3803},[242],"height:1.3211em;vertical-align:-0.3831em;",[],{"type":150,"tag":169,"props":3806,"children":3808},{"className":3807},[248],[3809,3814],{"type":150,"tag":169,"props":3810,"children":3812},{"className":3811},[248,489],[3813],{"type":155,"value":409},{"type":150,"tag":169,"props":3815,"children":3817},{"className":3816},[495],[3818],{"type":150,"tag":169,"props":3819,"children":3821},{"className":3820},[253,254],[3822,3851],{"type":150,"tag":169,"props":3823,"children":3825},{"className":3824},[259],[3826,3846],{"type":150,"tag":169,"props":3827,"children":3829},{"className":3828,"style":802},[264],[3830],{"type":150,"tag":169,"props":3831,"children":3832},{"style":512},[3833,3837],{"type":150,"tag":169,"props":3834,"children":3836},{"className":3835,"style":517},[274],[],{"type":150,"tag":169,"props":3838,"children":3840},{"className":3839},[522,523,524,525],[3841],{"type":150,"tag":169,"props":3842,"children":3844},{"className":3843},[248,489,525],[3845],{"type":155,"value":739},{"type":150,"tag":169,"props":3847,"children":3849},{"className":3848},[355],[3850],{"type":155,"value":358},{"type":150,"tag":169,"props":3852,"children":3854},{"className":3853},[259],[3855],{"type":150,"tag":169,"props":3856,"children":3858},{"className":3857,"style":544},[264],[3859],{"type":150,"tag":169,"props":3860,"children":3861},{},[],{"type":150,"tag":169,"props":3863,"children":3865},{"className":3864},[248],[3866,3872],{"type":150,"tag":169,"props":3867,"children":3870},{"className":3868,"style":3869},[248,489],"margin-right:0.1389em;",[3871],{"type":155,"value":3443},{"type":150,"tag":169,"props":3873,"children":3875},{"className":3874},[495],[3876],{"type":150,"tag":169,"props":3877,"children":3879},{"className":3878},[253,254],[3880,3942],{"type":150,"tag":169,"props":3881,"children":3883},{"className":3882},[259],[3884,3937],{"type":150,"tag":169,"props":3885,"children":3888},{"className":3886,"style":3887},[264],"height:0.938em;",[3889,3906],{"type":150,"tag":169,"props":3890,"children":3892},{"style":3891},"top:-2.453em;margin-left:-0.1389em;margin-right:0.05em;",[3893,3897],{"type":150,"tag":169,"props":3894,"children":3896},{"className":3895,"style":517},[274],[],{"type":150,"tag":169,"props":3898,"children":3900},{"className":3899},[522,523,524,525],[3901],{"type":150,"tag":169,"props":3902,"children":3904},{"className":3903,"style":2329},[248,489,525],[3905],{"type":155,"value":1845},{"type":150,"tag":169,"props":3907,"children":3909},{"style":3908},"top:-3.113em;margin-right:0.05em;",[3910,3914],{"type":150,"tag":169,"props":3911,"children":3913},{"className":3912,"style":517},[274],[],{"type":150,"tag":169,"props":3915,"children":3917},{"className":3916},[522,523,524,525],[3918],{"type":150,"tag":169,"props":3919,"children":3921},{"className":3920},[248,525],[3922,3927,3932],{"type":150,"tag":169,"props":3923,"children":3925},{"className":3924},[1004,525],[3926],{"type":155,"value":972},{"type":150,"tag":169,"props":3928,"children":3930},{"className":3929},[248,489,525],[3931],{"type":155,"value":977},{"type":150,"tag":169,"props":3933,"children":3935},{"className":3934},[1015,525],[3936],{"type":155,"value":982},{"type":150,"tag":169,"props":3938,"children":3940},{"className":3939},[355],[3941],{"type":155,"value":358},{"type":150,"tag":169,"props":3943,"children":3945},{"className":3944},[259],[3946],{"type":150,"tag":169,"props":3947,"children":3950},{"className":3948,"style":3949},[264],"height:0.3831em;",[3951],{"type":150,"tag":169,"props":3952,"children":3953},{},[],{"type":150,"tag":169,"props":3955,"children":3957},{"className":3956,"style":839},[558],[],{"type":150,"tag":169,"props":3959,"children":3961},{"className":3960},[844],[3962],{"type":155,"value":744},{"type":150,"tag":169,"props":3964,"children":3966},{"className":3965,"style":839},[558],[],{"type":150,"tag":169,"props":3968,"children":3970},{"className":3969},[237],[3971,3975,4079,4084,4089,4093,4180,4184,4189],{"type":150,"tag":169,"props":3972,"children":3974},{"className":3973,"style":3694},[242],[],{"type":150,"tag":169,"props":3976,"children":3978},{"className":3977},[248],[3979,3984],{"type":150,"tag":169,"props":3980,"children":3982},{"className":3981},[248,867],[3983],{"type":155,"value":754},{"type":150,"tag":169,"props":3985,"children":3987},{"className":3986},[495],[3988],{"type":150,"tag":169,"props":3989,"children":3991},{"className":3990},[253],[3992],{"type":150,"tag":169,"props":3993,"children":3995},{"className":3994},[259],[3996],{"type":150,"tag":169,"props":3997,"children":4000},{"className":3998,"style":3999},[264],"height:0.8991em;",[4001],{"type":150,"tag":169,"props":4002,"children":4003},{"style":3908},[4004,4008],{"type":150,"tag":169,"props":4005,"children":4007},{"className":4006,"style":517},[274],[],{"type":150,"tag":169,"props":4009,"children":4011},{"className":4010},[522,523,524,525],[4012],{"type":150,"tag":169,"props":4013,"children":4015},{"className":4014},[248,525],[4016],{"type":150,"tag":169,"props":4017,"children":4019},{"className":4018},[248,525],[4020,4025],{"type":150,"tag":169,"props":4021,"children":4023},{"className":4022},[248,489,525],[4024],{"type":155,"value":759},{"type":150,"tag":169,"props":4026,"children":4028},{"className":4027},[495],[4029],{"type":150,"tag":169,"props":4030,"children":4032},{"className":4031},[253,254],[4033,4067],{"type":150,"tag":169,"props":4034,"children":4036},{"className":4035},[259],[4037,4062],{"type":150,"tag":169,"props":4038,"children":4041},{"className":4039,"style":4040},[264],"height:0.3448em;",[4042],{"type":150,"tag":169,"props":4043,"children":4045},{"style":4044},"top:-2.3488em;margin-left:0em;margin-right:0.0714em;",[4046,4051],{"type":150,"tag":169,"props":4047,"children":4050},{"className":4048,"style":4049},[274],"height:2.5em;",[],{"type":150,"tag":169,"props":4052,"children":4056},{"className":4053},[522,4054,4055,525],"reset-size3","size1",[4057],{"type":150,"tag":169,"props":4058,"children":4060},{"className":4059,"style":2427},[248,489,525],[4061],{"type":155,"value":1876},{"type":150,"tag":169,"props":4063,"children":4065},{"className":4064},[355],[4066],{"type":155,"value":358},{"type":150,"tag":169,"props":4068,"children":4070},{"className":4069},[259],[4071],{"type":150,"tag":169,"props":4072,"children":4075},{"className":4073,"style":4074},[264],"height:0.1512em;",[4076],{"type":150,"tag":169,"props":4077,"children":4078},{},[],{"type":150,"tag":169,"props":4080,"children":4082},{"className":4081},[552],[4083],{"type":155,"value":421},{"type":150,"tag":169,"props":4085,"children":4088},{"className":4086,"style":4087},[558],"margin-right:1em;",[],{"type":150,"tag":169,"props":4090,"children":4092},{"className":4091,"style":559},[558],[],{"type":150,"tag":169,"props":4094,"children":4096},{"className":4095},[248],[4097,4102],{"type":150,"tag":169,"props":4098,"children":4100},{"className":4099,"style":2427},[248,489],[4101],{"type":155,"value":1876},{"type":150,"tag":169,"props":4103,"children":4105},{"className":4104},[495],[4106],{"type":150,"tag":169,"props":4107,"children":4109},{"className":4108},[253,254],[4110,4169],{"type":150,"tag":169,"props":4111,"children":4113},{"className":4112},[259],[4114,4164],{"type":150,"tag":169,"props":4115,"children":4117},{"className":4116,"style":1454},[264],[4118,4134],{"type":150,"tag":169,"props":4119,"children":4120},{"style":2898},[4121,4125],{"type":150,"tag":169,"props":4122,"children":4124},{"className":4123,"style":517},[274],[],{"type":150,"tag":169,"props":4126,"children":4128},{"className":4127},[522,523,524,525],[4129],{"type":150,"tag":169,"props":4130,"children":4132},{"className":4131},[248,489,525],[4133],{"type":155,"value":739},{"type":150,"tag":169,"props":4135,"children":4136},{"style":1475},[4137,4141],{"type":150,"tag":169,"props":4138,"children":4140},{"className":4139,"style":517},[274],[],{"type":150,"tag":169,"props":4142,"children":4144},{"className":4143},[522,523,524,525],[4145],{"type":150,"tag":169,"props":4146,"children":4148},{"className":4147},[248,525],[4149,4154,4159],{"type":150,"tag":169,"props":4150,"children":4152},{"className":4151},[1004,525],[4153],{"type":155,"value":972},{"type":150,"tag":169,"props":4155,"children":4157},{"className":4156},[248,489,525],[4158],{"type":155,"value":977},{"type":150,"tag":169,"props":4160,"children":4162},{"className":4161},[1015,525],[4163],{"type":155,"value":982},{"type":150,"tag":169,"props":4165,"children":4167},{"className":4166},[355],[4168],{"type":155,"value":358},{"type":150,"tag":169,"props":4170,"children":4172},{"className":4171},[259],[4173],{"type":150,"tag":169,"props":4174,"children":4176},{"className":4175,"style":2960},[264],[4177],{"type":150,"tag":169,"props":4178,"children":4179},{},[],{"type":150,"tag":169,"props":4181,"children":4183},{"className":4182,"style":839},[558],[],{"type":150,"tag":169,"props":4185,"children":4187},{"className":4186},[844],[4188],{"type":155,"value":1045},{"type":150,"tag":169,"props":4190,"children":4192},{"className":4191,"style":839},[558],[],{"type":150,"tag":169,"props":4194,"children":4196},{"className":4195},[237],[4197,4202,4259,4348,4352,4357],{"type":150,"tag":169,"props":4198,"children":4201},{"className":4199,"style":4200},[242],"height:1.3461em;vertical-align:-0.3013em;",[],{"type":150,"tag":169,"props":4203,"children":4205},{"className":4204},[248],[4206,4211],{"type":150,"tag":169,"props":4207,"children":4209},{"className":4208},[248,489],[4210],{"type":155,"value":409},{"type":150,"tag":169,"props":4212,"children":4214},{"className":4213},[495],[4215],{"type":150,"tag":169,"props":4216,"children":4218},{"className":4217},[253,254],[4219,4248],{"type":150,"tag":169,"props":4220,"children":4222},{"className":4221},[259],[4223,4243],{"type":150,"tag":169,"props":4224,"children":4226},{"className":4225,"style":802},[264],[4227],{"type":150,"tag":169,"props":4228,"children":4229},{"style":512},[4230,4234],{"type":150,"tag":169,"props":4231,"children":4233},{"className":4232,"style":517},[274],[],{"type":150,"tag":169,"props":4235,"children":4237},{"className":4236},[522,523,524,525],[4238],{"type":150,"tag":169,"props":4239,"children":4241},{"className":4240},[248,489,525],[4242],{"type":155,"value":739},{"type":150,"tag":169,"props":4244,"children":4246},{"className":4245},[355],[4247],{"type":155,"value":358},{"type":150,"tag":169,"props":4249,"children":4251},{"className":4250},[259],[4252],{"type":150,"tag":169,"props":4253,"children":4255},{"className":4254,"style":544},[264],[4256],{"type":150,"tag":169,"props":4257,"children":4258},{},[],{"type":150,"tag":169,"props":4260,"children":4262},{"className":4261},[248],[4263,4268],{"type":150,"tag":169,"props":4264,"children":4266},{"className":4265,"style":3869},[248,489],[4267],{"type":155,"value":3443},{"type":150,"tag":169,"props":4269,"children":4271},{"className":4270},[495],[4272],{"type":150,"tag":169,"props":4273,"children":4275},{"className":4274},[253,254],[4276,4336],{"type":150,"tag":169,"props":4277,"children":4279},{"className":4278},[259],[4280,4331],{"type":150,"tag":169,"props":4281,"children":4283},{"className":4282,"style":1454},[264],[4284,4301],{"type":150,"tag":169,"props":4285,"children":4287},{"style":4286},"top:-2.3987em;margin-left:-0.1389em;margin-right:0.05em;",[4288,4292],{"type":150,"tag":169,"props":4289,"children":4291},{"className":4290,"style":517},[274],[],{"type":150,"tag":169,"props":4293,"children":4295},{"className":4294},[522,523,524,525],[4296],{"type":150,"tag":169,"props":4297,"children":4299},{"className":4298,"style":2427},[248,489,525],[4300],{"type":155,"value":1876},{"type":150,"tag":169,"props":4302,"children":4303},{"style":1475},[4304,4308],{"type":150,"tag":169,"props":4305,"children":4307},{"className":4306,"style":517},[274],[],{"type":150,"tag":169,"props":4309,"children":4311},{"className":4310},[522,523,524,525],[4312],{"type":150,"tag":169,"props":4313,"children":4315},{"className":4314},[248,525],[4316,4321,4326],{"type":150,"tag":169,"props":4317,"children":4319},{"className":4318},[1004,525],[4320],{"type":155,"value":972},{"type":150,"tag":169,"props":4322,"children":4324},{"className":4323},[248,489,525],[4325],{"type":155,"value":977},{"type":150,"tag":169,"props":4327,"children":4329},{"className":4328},[1015,525],[4330],{"type":155,"value":982},{"type":150,"tag":169,"props":4332,"children":4334},{"className":4333},[355],[4335],{"type":155,"value":358},{"type":150,"tag":169,"props":4337,"children":4339},{"className":4338},[259],[4340],{"type":150,"tag":169,"props":4341,"children":4344},{"className":4342,"style":4343},[264],"height:0.3013em;",[4345],{"type":150,"tag":169,"props":4346,"children":4347},{},[],{"type":150,"tag":169,"props":4349,"children":4351},{"className":4350,"style":839},[558],[],{"type":150,"tag":169,"props":4353,"children":4355},{"className":4354},[844],[4356],{"type":155,"value":744},{"type":150,"tag":169,"props":4358,"children":4360},{"className":4359,"style":839},[558],[],{"type":150,"tag":169,"props":4362,"children":4364},{"className":4363},[237],[4365,4369,4466,4471,4475,4479,4566,4570,4575],{"type":150,"tag":169,"props":4366,"children":4368},{"className":4367,"style":3694},[242],[],{"type":150,"tag":169,"props":4370,"children":4372},{"className":4371},[248],[4373,4378],{"type":150,"tag":169,"props":4374,"children":4376},{"className":4375},[248,867],[4377],{"type":155,"value":754},{"type":150,"tag":169,"props":4379,"children":4381},{"className":4380},[495],[4382],{"type":150,"tag":169,"props":4383,"children":4385},{"className":4384},[253],[4386],{"type":150,"tag":169,"props":4387,"children":4389},{"className":4388},[259],[4390],{"type":150,"tag":169,"props":4391,"children":4393},{"className":4392,"style":3999},[264],[4394],{"type":150,"tag":169,"props":4395,"children":4396},{"style":3908},[4397,4401],{"type":150,"tag":169,"props":4398,"children":4400},{"className":4399,"style":517},[274],[],{"type":150,"tag":169,"props":4402,"children":4404},{"className":4403},[522,523,524,525],[4405],{"type":150,"tag":169,"props":4406,"children":4408},{"className":4407},[248,525],[4409],{"type":150,"tag":169,"props":4410,"children":4412},{"className":4411},[248,525],[4413,4418],{"type":150,"tag":169,"props":4414,"children":4416},{"className":4415},[248,489,525],[4417],{"type":155,"value":759},{"type":150,"tag":169,"props":4419,"children":4421},{"className":4420},[495],[4422],{"type":150,"tag":169,"props":4423,"children":4425},{"className":4424},[253,254],[4426,4455],{"type":150,"tag":169,"props":4427,"children":4429},{"className":4428},[259],[4430,4450],{"type":150,"tag":169,"props":4431,"children":4433},{"className":4432,"style":4040},[264],[4434],{"type":150,"tag":169,"props":4435,"children":4436},{"style":4044},[4437,4441],{"type":150,"tag":169,"props":4438,"children":4440},{"className":4439,"style":4049},[274],[],{"type":150,"tag":169,"props":4442,"children":4444},{"className":4443},[522,4054,4055,525],[4445],{"type":150,"tag":169,"props":4446,"children":4448},{"className":4447,"style":2427},[248,489,525],[4449],{"type":155,"value":1876},{"type":150,"tag":169,"props":4451,"children":4453},{"className":4452},[355],[4454],{"type":155,"value":358},{"type":150,"tag":169,"props":4456,"children":4458},{"className":4457},[259],[4459],{"type":150,"tag":169,"props":4460,"children":4462},{"className":4461,"style":4074},[264],[4463],{"type":150,"tag":169,"props":4464,"children":4465},{},[],{"type":150,"tag":169,"props":4467,"children":4469},{"className":4468},[552],[4470],{"type":155,"value":421},{"type":150,"tag":169,"props":4472,"children":4474},{"className":4473,"style":4087},[558],[],{"type":150,"tag":169,"props":4476,"children":4478},{"className":4477,"style":559},[558],[],{"type":150,"tag":169,"props":4480,"children":4482},{"className":4481},[248],[4483,4488],{"type":150,"tag":169,"props":4484,"children":4486},{"className":4485,"style":2329},[248,489],[4487],{"type":155,"value":1915},{"type":150,"tag":169,"props":4489,"children":4491},{"className":4490},[495],[4492],{"type":150,"tag":169,"props":4493,"children":4495},{"className":4494},[253,254],[4496,4555],{"type":150,"tag":169,"props":4497,"children":4499},{"className":4498},[259],[4500,4550],{"type":150,"tag":169,"props":4501,"children":4503},{"className":4502,"style":1454},[264],[4504,4520],{"type":150,"tag":169,"props":4505,"children":4506},{"style":3292},[4507,4511],{"type":150,"tag":169,"props":4508,"children":4510},{"className":4509,"style":517},[274],[],{"type":150,"tag":169,"props":4512,"children":4514},{"className":4513},[522,523,524,525],[4515],{"type":150,"tag":169,"props":4516,"children":4518},{"className":4517},[248,489,525],[4519],{"type":155,"value":739},{"type":150,"tag":169,"props":4521,"children":4522},{"style":1475},[4523,4527],{"type":150,"tag":169,"props":4524,"children":4526},{"className":4525,"style":517},[274],[],{"type":150,"tag":169,"props":4528,"children":4530},{"className":4529},[522,523,524,525],[4531],{"type":150,"tag":169,"props":4532,"children":4534},{"className":4533},[248,525],[4535,4540,4545],{"type":150,"tag":169,"props":4536,"children":4538},{"className":4537},[1004,525],[4539],{"type":155,"value":972},{"type":150,"tag":169,"props":4541,"children":4543},{"className":4542},[248,489,525],[4544],{"type":155,"value":977},{"type":150,"tag":169,"props":4546,"children":4548},{"className":4547},[1015,525],[4549],{"type":155,"value":982},{"type":150,"tag":169,"props":4551,"children":4553},{"className":4552},[355],[4554],{"type":155,"value":358},{"type":150,"tag":169,"props":4556,"children":4558},{"className":4557},[259],[4559],{"type":150,"tag":169,"props":4560,"children":4562},{"className":4561,"style":2960},[264],[4563],{"type":150,"tag":169,"props":4564,"children":4565},{},[],{"type":150,"tag":169,"props":4567,"children":4569},{"className":4568,"style":839},[558],[],{"type":150,"tag":169,"props":4571,"children":4573},{"className":4572},[844],[4574],{"type":155,"value":1045},{"type":150,"tag":169,"props":4576,"children":4578},{"className":4577,"style":839},[558],[],{"type":150,"tag":169,"props":4580,"children":4582},{"className":4581},[237],[4583,4588,4645,4733,4737,4742],{"type":150,"tag":169,"props":4584,"children":4587},{"className":4585,"style":4586},[242],"height:1.185em;vertical-align:-0.247em;",[],{"type":150,"tag":169,"props":4589,"children":4591},{"className":4590},[248],[4592,4597],{"type":150,"tag":169,"props":4593,"children":4595},{"className":4594},[248,489],[4596],{"type":155,"value":409},{"type":150,"tag":169,"props":4598,"children":4600},{"className":4599},[495],[4601],{"type":150,"tag":169,"props":4602,"children":4604},{"className":4603},[253,254],[4605,4634],{"type":150,"tag":169,"props":4606,"children":4608},{"className":4607},[259],[4609,4629],{"type":150,"tag":169,"props":4610,"children":4612},{"className":4611,"style":802},[264],[4613],{"type":150,"tag":169,"props":4614,"children":4615},{"style":512},[4616,4620],{"type":150,"tag":169,"props":4617,"children":4619},{"className":4618,"style":517},[274],[],{"type":150,"tag":169,"props":4621,"children":4623},{"className":4622},[522,523,524,525],[4624],{"type":150,"tag":169,"props":4625,"children":4627},{"className":4626},[248,489,525],[4628],{"type":155,"value":739},{"type":150,"tag":169,"props":4630,"children":4632},{"className":4631},[355],[4633],{"type":155,"value":358},{"type":150,"tag":169,"props":4635,"children":4637},{"className":4636},[259],[4638],{"type":150,"tag":169,"props":4639,"children":4641},{"className":4640,"style":544},[264],[4642],{"type":150,"tag":169,"props":4643,"children":4644},{},[],{"type":150,"tag":169,"props":4646,"children":4648},{"className":4647},[248],[4649,4654],{"type":150,"tag":169,"props":4650,"children":4652},{"className":4651,"style":3869},[248,489],[4653],{"type":155,"value":3443},{"type":150,"tag":169,"props":4655,"children":4657},{"className":4656},[495],[4658],{"type":150,"tag":169,"props":4659,"children":4661},{"className":4660},[253,254],[4662,4721],{"type":150,"tag":169,"props":4663,"children":4665},{"className":4664},[259],[4666,4716],{"type":150,"tag":169,"props":4667,"children":4669},{"className":4668,"style":3887},[264],[4670,4686],{"type":150,"tag":169,"props":4671,"children":4672},{"style":3891},[4673,4677],{"type":150,"tag":169,"props":4674,"children":4676},{"className":4675,"style":517},[274],[],{"type":150,"tag":169,"props":4678,"children":4680},{"className":4679},[522,523,524,525],[4681],{"type":150,"tag":169,"props":4682,"children":4684},{"className":4683,"style":2329},[248,489,525],[4685],{"type":155,"value":1915},{"type":150,"tag":169,"props":4687,"children":4688},{"style":3908},[4689,4693],{"type":150,"tag":169,"props":4690,"children":4692},{"className":4691,"style":517},[274],[],{"type":150,"tag":169,"props":4694,"children":4696},{"className":4695},[522,523,524,525],[4697],{"type":150,"tag":169,"props":4698,"children":4700},{"className":4699},[248,525],[4701,4706,4711],{"type":150,"tag":169,"props":4702,"children":4704},{"className":4703},[1004,525],[4705],{"type":155,"value":972},{"type":150,"tag":169,"props":4707,"children":4709},{"className":4708},[248,489,525],[4710],{"type":155,"value":977},{"type":150,"tag":169,"props":4712,"children":4714},{"className":4713},[1015,525],[4715],{"type":155,"value":982},{"type":150,"tag":169,"props":4717,"children":4719},{"className":4718},[355],[4720],{"type":155,"value":358},{"type":150,"tag":169,"props":4722,"children":4724},{"className":4723},[259],[4725],{"type":150,"tag":169,"props":4726,"children":4729},{"className":4727,"style":4728},[264],"height:0.247em;",[4730],{"type":150,"tag":169,"props":4731,"children":4732},{},[],{"type":150,"tag":169,"props":4734,"children":4736},{"className":4735,"style":839},[558],[],{"type":150,"tag":169,"props":4738,"children":4740},{"className":4739},[844],[4741],{"type":155,"value":744},{"type":150,"tag":169,"props":4743,"children":4745},{"className":4744,"style":839},[558],[],{"type":150,"tag":169,"props":4747,"children":4749},{"className":4748},[237],[4750,4754],{"type":150,"tag":169,"props":4751,"children":4753},{"className":4752,"style":3999},[242],[],{"type":150,"tag":169,"props":4755,"children":4757},{"className":4756},[248],[4758,4763],{"type":150,"tag":169,"props":4759,"children":4761},{"className":4760},[248,867],[4762],{"type":155,"value":754},{"type":150,"tag":169,"props":4764,"children":4766},{"className":4765},[495],[4767],{"type":150,"tag":169,"props":4768,"children":4770},{"className":4769},[253],[4771],{"type":150,"tag":169,"props":4772,"children":4774},{"className":4773},[259],[4775],{"type":150,"tag":169,"props":4776,"children":4778},{"className":4777,"style":3999},[264],[4779],{"type":150,"tag":169,"props":4780,"children":4781},{"style":3908},[4782,4786],{"type":150,"tag":169,"props":4783,"children":4785},{"className":4784,"style":517},[274],[],{"type":150,"tag":169,"props":4787,"children":4789},{"className":4788},[522,523,524,525],[4790],{"type":150,"tag":169,"props":4791,"children":4793},{"className":4792},[248,525],[4794],{"type":150,"tag":169,"props":4795,"children":4797},{"className":4796},[248,525],[4798,4803],{"type":150,"tag":169,"props":4799,"children":4801},{"className":4800},[248,489,525],[4802],{"type":155,"value":759},{"type":150,"tag":169,"props":4804,"children":4806},{"className":4805},[495],[4807],{"type":150,"tag":169,"props":4808,"children":4810},{"className":4809},[253,254],[4811,4842],{"type":150,"tag":169,"props":4812,"children":4814},{"className":4813},[259],[4815,4837],{"type":150,"tag":169,"props":4816,"children":4819},{"className":4817,"style":4818},[264],"height:0.1645em;",[4820],{"type":150,"tag":169,"props":4821,"children":4823},{"style":4822},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[4824,4828],{"type":150,"tag":169,"props":4825,"children":4827},{"className":4826,"style":4049},[274],[],{"type":150,"tag":169,"props":4829,"children":4831},{"className":4830},[522,4054,4055,525],[4832],{"type":150,"tag":169,"props":4833,"children":4835},{"className":4834,"style":2329},[248,489,525],[4836],{"type":155,"value":1915},{"type":150,"tag":169,"props":4838,"children":4840},{"className":4839},[355],[4841],{"type":155,"value":358},{"type":150,"tag":169,"props":4843,"children":4845},{"className":4844},[259],[4846],{"type":150,"tag":169,"props":4847,"children":4850},{"className":4848,"style":4849},[264],"height:0.143em;",[4851],{"type":150,"tag":169,"props":4852,"children":4853},{},[],{"type":150,"tag":151,"props":4855,"children":4856},{},[4857,4859,5109,5111,5193,5195,5358,5360,5523],{"type":155,"value":4858},"（省略了缩放因子。常见设置是 ",{"type":150,"tag":169,"props":4860,"children":4862},{"className":4861},[177],[4863,4924],{"type":150,"tag":169,"props":4864,"children":4866},{"className":4865},[182],[4867],{"type":150,"tag":185,"props":4868,"children":4869},{"xmlns":187},[4870],{"type":150,"tag":191,"props":4871,"children":4872},{},[4873,4919],{"type":150,"tag":195,"props":4874,"children":4875},{},[4876,4887,4891,4902,4906,4910,4915],{"type":150,"tag":401,"props":4877,"children":4878},{},[4879,4883],{"type":150,"tag":405,"props":4880,"children":4881},{},[4882],{"type":155,"value":759},{"type":150,"tag":405,"props":4884,"children":4885},{},[4886],{"type":155,"value":1876},{"type":150,"tag":417,"props":4888,"children":4889},{},[4890],{"type":155,"value":1045},{"type":150,"tag":401,"props":4892,"children":4893},{},[4894,4898],{"type":150,"tag":405,"props":4895,"children":4896},{},[4897],{"type":155,"value":759},{"type":150,"tag":405,"props":4899,"children":4900},{},[4901],{"type":155,"value":1915},{"type":150,"tag":417,"props":4903,"children":4904},{},[4905],{"type":155,"value":1045},{"type":150,"tag":405,"props":4907,"children":4908},{},[4909],{"type":155,"value":759},{"type":150,"tag":405,"props":4911,"children":4912},{"mathvariant":1796},[4913],{"type":155,"value":4914},"\u002F",{"type":150,"tag":405,"props":4916,"children":4917},{},[4918],{"type":155,"value":926},{"type":150,"tag":223,"props":4920,"children":4921},{"encoding":225},[4922],{"type":155,"value":4923},"d_k = d_v = d\u002Fh",{"type":150,"tag":169,"props":4925,"children":4927},{"className":4926,"ariaHidden":214},[232],[4928,5007,5086],{"type":150,"tag":169,"props":4929,"children":4931},{"className":4930},[237],[4932,4937,4994,4998,5003],{"type":150,"tag":169,"props":4933,"children":4936},{"className":4934,"style":4935},[242],"height:0.8444em;vertical-align:-0.15em;",[],{"type":150,"tag":169,"props":4938,"children":4940},{"className":4939},[248],[4941,4946],{"type":150,"tag":169,"props":4942,"children":4944},{"className":4943},[248,489],[4945],{"type":155,"value":759},{"type":150,"tag":169,"props":4947,"children":4949},{"className":4948},[495],[4950],{"type":150,"tag":169,"props":4951,"children":4953},{"className":4952},[253,254],[4954,4983],{"type":150,"tag":169,"props":4955,"children":4957},{"className":4956},[259],[4958,4978],{"type":150,"tag":169,"props":4959,"children":4961},{"className":4960,"style":674},[264],[4962],{"type":150,"tag":169,"props":4963,"children":4964},{"style":512},[4965,4969],{"type":150,"tag":169,"props":4966,"children":4968},{"className":4967,"style":517},[274],[],{"type":150,"tag":169,"props":4970,"children":4972},{"className":4971},[522,523,524,525],[4973],{"type":150,"tag":169,"props":4974,"children":4976},{"className":4975,"style":2427},[248,489,525],[4977],{"type":155,"value":1876},{"type":150,"tag":169,"props":4979,"children":4981},{"className":4980},[355],[4982],{"type":155,"value":358},{"type":150,"tag":169,"props":4984,"children":4986},{"className":4985},[259],[4987],{"type":150,"tag":169,"props":4988,"children":4990},{"className":4989,"style":544},[264],[4991],{"type":150,"tag":169,"props":4992,"children":4993},{},[],{"type":150,"tag":169,"props":4995,"children":4997},{"className":4996,"style":839},[558],[],{"type":150,"tag":169,"props":4999,"children":5001},{"className":5000},[844],[5002],{"type":155,"value":1045},{"type":150,"tag":169,"props":5004,"children":5006},{"className":5005,"style":839},[558],[],{"type":150,"tag":169,"props":5008,"children":5010},{"className":5009},[237],[5011,5015,5073,5077,5082],{"type":150,"tag":169,"props":5012,"children":5014},{"className":5013,"style":4935},[242],[],{"type":150,"tag":169,"props":5016,"children":5018},{"className":5017},[248],[5019,5024],{"type":150,"tag":169,"props":5020,"children":5022},{"className":5021},[248,489],[5023],{"type":155,"value":759},{"type":150,"tag":169,"props":5025,"children":5027},{"className":5026},[495],[5028],{"type":150,"tag":169,"props":5029,"children":5031},{"className":5030},[253,254],[5032,5062],{"type":150,"tag":169,"props":5033,"children":5035},{"className":5034},[259],[5036,5057],{"type":150,"tag":169,"props":5037,"children":5040},{"className":5038,"style":5039},[264],"height:0.1514em;",[5041],{"type":150,"tag":169,"props":5042,"children":5043},{"style":512},[5044,5048],{"type":150,"tag":169,"props":5045,"children":5047},{"className":5046,"style":517},[274],[],{"type":150,"tag":169,"props":5049,"children":5051},{"className":5050},[522,523,524,525],[5052],{"type":150,"tag":169,"props":5053,"children":5055},{"className":5054,"style":2329},[248,489,525],[5056],{"type":155,"value":1915},{"type":150,"tag":169,"props":5058,"children":5060},{"className":5059},[355],[5061],{"type":155,"value":358},{"type":150,"tag":169,"props":5063,"children":5065},{"className":5064},[259],[5066],{"type":150,"tag":169,"props":5067,"children":5069},{"className":5068,"style":544},[264],[5070],{"type":150,"tag":169,"props":5071,"children":5072},{},[],{"type":150,"tag":169,"props":5074,"children":5076},{"className":5075,"style":839},[558],[],{"type":150,"tag":169,"props":5078,"children":5080},{"className":5079},[844],[5081],{"type":155,"value":1045},{"type":150,"tag":169,"props":5083,"children":5085},{"className":5084,"style":839},[558],[],{"type":150,"tag":169,"props":5087,"children":5089},{"className":5088},[237],[5090,5094,5099,5104],{"type":150,"tag":169,"props":5091,"children":5093},{"className":5092,"style":999},[242],[],{"type":150,"tag":169,"props":5095,"children":5097},{"className":5096},[248,489],[5098],{"type":155,"value":759},{"type":150,"tag":169,"props":5100,"children":5102},{"className":5101},[248],[5103],{"type":155,"value":4914},{"type":150,"tag":169,"props":5105,"children":5107},{"className":5106},[248,489],[5108],{"type":155,"value":926},{"type":155,"value":5110},"。）关键在因果性。自回归生成时，新预测出来的第 ",{"type":150,"tag":169,"props":5112,"children":5114},{"className":5113},[177],[5115,5146],{"type":150,"tag":169,"props":5116,"children":5118},{"className":5117},[182],[5119],{"type":150,"tag":185,"props":5120,"children":5121},{"xmlns":187},[5122],{"type":150,"tag":191,"props":5123,"children":5124},{},[5125,5141],{"type":150,"tag":195,"props":5126,"children":5127},{},[5128,5132,5137],{"type":150,"tag":405,"props":5129,"children":5130},{},[5131],{"type":155,"value":1213},{"type":150,"tag":417,"props":5133,"children":5134},{},[5135],{"type":155,"value":5136},"+",{"type":150,"tag":411,"props":5138,"children":5139},{},[5140],{"type":155,"value":415},{"type":150,"tag":223,"props":5142,"children":5143},{"encoding":225},[5144],{"type":155,"value":5145},"t+1",{"type":150,"tag":169,"props":5147,"children":5149},{"className":5148,"ariaHidden":214},[232],[5150,5179],{"type":150,"tag":169,"props":5151,"children":5153},{"className":5152},[237],[5154,5159,5164,5169,5175],{"type":150,"tag":169,"props":5155,"children":5158},{"className":5156,"style":5157},[242],"height:0.6984em;vertical-align:-0.0833em;",[],{"type":150,"tag":169,"props":5160,"children":5162},{"className":5161},[248,489],[5163],{"type":155,"value":1213},{"type":150,"tag":169,"props":5165,"children":5168},{"className":5166,"style":5167},[558],"margin-right:0.2222em;",[],{"type":150,"tag":169,"props":5170,"children":5173},{"className":5171},[5172],"mbin",[5174],{"type":155,"value":5136},{"type":150,"tag":169,"props":5176,"children":5178},{"className":5177,"style":5167},[558],[],{"type":150,"tag":169,"props":5180,"children":5182},{"className":5181},[237],[5183,5188],{"type":150,"tag":169,"props":5184,"children":5187},{"className":5185,"style":5186},[242],"height:0.6444em;",[],{"type":150,"tag":169,"props":5189,"children":5191},{"className":5190},[248],[5192],{"type":155,"value":415},{"type":155,"value":5194}," 个 token 不会影响已经算好的 ",{"type":150,"tag":169,"props":5196,"children":5198},{"className":5197},[177],[5199,5250],{"type":150,"tag":169,"props":5200,"children":5202},{"className":5201},[182],[5203],{"type":150,"tag":185,"props":5204,"children":5205},{"xmlns":187},[5206],{"type":150,"tag":191,"props":5207,"children":5208},{},[5209,5245],{"type":150,"tag":195,"props":5210,"children":5211},{},[5212],{"type":150,"tag":1224,"props":5213,"children":5214},{},[5215,5219,5230],{"type":150,"tag":405,"props":5216,"children":5217},{},[5218],{"type":155,"value":1876},{"type":150,"tag":195,"props":5220,"children":5221},{},[5222,5226],{"type":150,"tag":417,"props":5223,"children":5224},{},[5225],{"type":155,"value":1884},{"type":150,"tag":405,"props":5227,"children":5228},{},[5229],{"type":155,"value":1213},{"type":150,"tag":195,"props":5231,"children":5232},{},[5233,5237,5241],{"type":150,"tag":417,"props":5234,"children":5235},{"stretchy":207},[5236],{"type":155,"value":972},{"type":150,"tag":405,"props":5238,"children":5239},{},[5240],{"type":155,"value":977},{"type":150,"tag":417,"props":5242,"children":5243},{"stretchy":207},[5244],{"type":155,"value":982},{"type":150,"tag":223,"props":5246,"children":5247},{"encoding":225},[5248],{"type":155,"value":5249},"k^{(s)}_{\\le t}",{"type":150,"tag":169,"props":5251,"children":5253},{"className":5252,"ariaHidden":214},[232],[5254],{"type":150,"tag":169,"props":5255,"children":5257},{"className":5256},[237],[5258,5262],{"type":150,"tag":169,"props":5259,"children":5261},{"className":5260,"style":2305},[242],[],{"type":150,"tag":169,"props":5263,"children":5265},{"className":5264},[248],[5266,5271],{"type":150,"tag":169,"props":5267,"children":5269},{"className":5268,"style":2427},[248,489],[5270],{"type":155,"value":1876},{"type":150,"tag":169,"props":5272,"children":5274},{"className":5273},[495],[5275],{"type":150,"tag":169,"props":5276,"children":5278},{"className":5277},[253,254],[5279,5347],{"type":150,"tag":169,"props":5280,"children":5282},{"className":5281},[259],[5283,5342],{"type":150,"tag":169,"props":5284,"children":5286},{"className":5285,"style":1454},[264],[5287,5312],{"type":150,"tag":169,"props":5288,"children":5289},{"style":2448},[5290,5294],{"type":150,"tag":169,"props":5291,"children":5293},{"className":5292,"style":517},[274],[],{"type":150,"tag":169,"props":5295,"children":5297},{"className":5296},[522,523,524,525],[5298],{"type":150,"tag":169,"props":5299,"children":5301},{"className":5300},[248,525],[5302,5307],{"type":150,"tag":169,"props":5303,"children":5305},{"className":5304},[844,525],[5306],{"type":155,"value":1884},{"type":150,"tag":169,"props":5308,"children":5310},{"className":5309},[248,489,525],[5311],{"type":155,"value":1213},{"type":150,"tag":169,"props":5313,"children":5314},{"style":1475},[5315,5319],{"type":150,"tag":169,"props":5316,"children":5318},{"className":5317,"style":517},[274],[],{"type":150,"tag":169,"props":5320,"children":5322},{"className":5321},[522,523,524,525],[5323],{"type":150,"tag":169,"props":5324,"children":5326},{"className":5325},[248,525],[5327,5332,5337],{"type":150,"tag":169,"props":5328,"children":5330},{"className":5329},[1004,525],[5331],{"type":155,"value":972},{"type":150,"tag":169,"props":5333,"children":5335},{"className":5334},[248,489,525],[5336],{"type":155,"value":977},{"type":150,"tag":169,"props":5338,"children":5340},{"className":5339},[1015,525],[5341],{"type":155,"value":982},{"type":150,"tag":169,"props":5343,"children":5345},{"className":5344},[355],[5346],{"type":155,"value":358},{"type":150,"tag":169,"props":5348,"children":5350},{"className":5349},[259],[5351],{"type":150,"tag":169,"props":5352,"children":5354},{"className":5353,"style":2514},[264],[5355],{"type":150,"tag":169,"props":5356,"children":5357},{},[],{"type":155,"value":5359}," 和 ",{"type":150,"tag":169,"props":5361,"children":5363},{"className":5362},[177],[5364,5415],{"type":150,"tag":169,"props":5365,"children":5367},{"className":5366},[182],[5368],{"type":150,"tag":185,"props":5369,"children":5370},{"xmlns":187},[5371],{"type":150,"tag":191,"props":5372,"children":5373},{},[5374,5410],{"type":150,"tag":195,"props":5375,"children":5376},{},[5377],{"type":150,"tag":1224,"props":5378,"children":5379},{},[5380,5384,5395],{"type":150,"tag":405,"props":5381,"children":5382},{},[5383],{"type":155,"value":1915},{"type":150,"tag":195,"props":5385,"children":5386},{},[5387,5391],{"type":150,"tag":417,"props":5388,"children":5389},{},[5390],{"type":155,"value":1884},{"type":150,"tag":405,"props":5392,"children":5393},{},[5394],{"type":155,"value":1213},{"type":150,"tag":195,"props":5396,"children":5397},{},[5398,5402,5406],{"type":150,"tag":417,"props":5399,"children":5400},{"stretchy":207},[5401],{"type":155,"value":972},{"type":150,"tag":405,"props":5403,"children":5404},{},[5405],{"type":155,"value":977},{"type":150,"tag":417,"props":5407,"children":5408},{"stretchy":207},[5409],{"type":155,"value":982},{"type":150,"tag":223,"props":5411,"children":5412},{"encoding":225},[5413],{"type":155,"value":5414},"v^{(s)}_{\\le t}",{"type":150,"tag":169,"props":5416,"children":5418},{"className":5417,"ariaHidden":214},[232],[5419],{"type":150,"tag":169,"props":5420,"children":5422},{"className":5421},[237],[5423,5427],{"type":150,"tag":169,"props":5424,"children":5426},{"className":5425,"style":2305},[242],[],{"type":150,"tag":169,"props":5428,"children":5430},{"className":5429},[248],[5431,5436],{"type":150,"tag":169,"props":5432,"children":5434},{"className":5433,"style":2329},[248,489],[5435],{"type":155,"value":1915},{"type":150,"tag":169,"props":5437,"children":5439},{"className":5438},[495],[5440],{"type":150,"tag":169,"props":5441,"children":5443},{"className":5442},[253,254],[5444,5512],{"type":150,"tag":169,"props":5445,"children":5447},{"className":5446},[259],[5448,5507],{"type":150,"tag":169,"props":5449,"children":5451},{"className":5450,"style":1454},[264],[5452,5477],{"type":150,"tag":169,"props":5453,"children":5454},{"style":2555},[5455,5459],{"type":150,"tag":169,"props":5456,"children":5458},{"className":5457,"style":517},[274],[],{"type":150,"tag":169,"props":5460,"children":5462},{"className":5461},[522,523,524,525],[5463],{"type":150,"tag":169,"props":5464,"children":5466},{"className":5465},[248,525],[5467,5472],{"type":150,"tag":169,"props":5468,"children":5470},{"className":5469},[844,525],[5471],{"type":155,"value":1884},{"type":150,"tag":169,"props":5473,"children":5475},{"className":5474},[248,489,525],[5476],{"type":155,"value":1213},{"type":150,"tag":169,"props":5478,"children":5479},{"style":1475},[5480,5484],{"type":150,"tag":169,"props":5481,"children":5483},{"className":5482,"style":517},[274],[],{"type":150,"tag":169,"props":5485,"children":5487},{"className":5486},[522,523,524,525],[5488],{"type":150,"tag":169,"props":5489,"children":5491},{"className":5490},[248,525],[5492,5497,5502],{"type":150,"tag":169,"props":5493,"children":5495},{"className":5494},[1004,525],[5496],{"type":155,"value":972},{"type":150,"tag":169,"props":5498,"children":5500},{"className":5499},[248,489,525],[5501],{"type":155,"value":977},{"type":150,"tag":169,"props":5503,"children":5505},{"className":5504},[1015,525],[5506],{"type":155,"value":982},{"type":150,"tag":169,"props":5508,"children":5510},{"className":5509},[355],[5511],{"type":155,"value":358},{"type":150,"tag":169,"props":5513,"children":5515},{"className":5514},[259],[5516],{"type":150,"tag":169,"props":5517,"children":5519},{"className":5518,"style":2514},[264],[5520],{"type":150,"tag":169,"props":5521,"children":5522},{},[],{"type":155,"value":5524},"，所以这部分可以缓存下来，避免重复计算。这就是 KV Cache。它的大小约等于：",{"type":150,"tag":169,"props":5526,"children":5528},{"className":5527},[172],[5529],{"type":150,"tag":169,"props":5530,"children":5532},{"className":5531},[177],[5533,5606],{"type":150,"tag":169,"props":5534,"children":5536},{"className":5535},[182],[5537],{"type":150,"tag":185,"props":5538,"children":5539},{"xmlns":187,"display":188},[5540],{"type":150,"tag":191,"props":5541,"children":5542},{},[5543,5601],{"type":150,"tag":195,"props":5544,"children":5545},{},[5546,5550,5555,5560,5564,5568,5572,5583,5587,5592,5596],{"type":150,"tag":411,"props":5547,"children":5548},{},[5549],{"type":155,"value":433},{"type":150,"tag":417,"props":5551,"children":5552},{},[5553],{"type":155,"value":5554},"×",{"type":150,"tag":405,"props":5556,"children":5557},{},[5558],{"type":155,"value":5559},"L",{"type":150,"tag":417,"props":5561,"children":5562},{},[5563],{"type":155,"value":5554},{"type":150,"tag":405,"props":5565,"children":5566},{},[5567],{"type":155,"value":926},{"type":150,"tag":417,"props":5569,"children":5570},{},[5571],{"type":155,"value":5554},{"type":150,"tag":401,"props":5573,"children":5574},{},[5575,5579],{"type":150,"tag":405,"props":5576,"children":5577},{},[5578],{"type":155,"value":759},{"type":150,"tag":405,"props":5580,"children":5581},{},[5582],{"type":155,"value":1876},{"type":150,"tag":417,"props":5584,"children":5585},{},[5586],{"type":155,"value":5554},{"type":150,"tag":405,"props":5588,"children":5589},{},[5590],{"type":155,"value":5591},"N",{"type":150,"tag":417,"props":5593,"children":5594},{},[5595],{"type":155,"value":5554},{"type":150,"tag":217,"props":5597,"children":5598},{},[5599],{"type":155,"value":5600},"bytes",{"type":150,"tag":223,"props":5602,"children":5603},{"encoding":225},[5604],{"type":155,"value":5605},"2 \\times L \\times h \\times d_k \\times N \\times \\text{bytes}",{"type":150,"tag":169,"props":5607,"children":5609},{"className":5608,"ariaHidden":214},[232],[5610,5637,5664,5691,5769,5796],{"type":150,"tag":169,"props":5611,"children":5613},{"className":5612},[237],[5614,5619,5624,5628,5633],{"type":150,"tag":169,"props":5615,"children":5618},{"className":5616,"style":5617},[242],"height:0.7278em;vertical-align:-0.0833em;",[],{"type":150,"tag":169,"props":5620,"children":5622},{"className":5621},[248],[5623],{"type":155,"value":433},{"type":150,"tag":169,"props":5625,"children":5627},{"className":5626,"style":5167},[558],[],{"type":150,"tag":169,"props":5629,"children":5631},{"className":5630},[5172],[5632],{"type":155,"value":5554},{"type":150,"tag":169,"props":5634,"children":5636},{"className":5635,"style":5167},[558],[],{"type":150,"tag":169,"props":5638,"children":5640},{"className":5639},[237],[5641,5646,5651,5655,5660],{"type":150,"tag":169,"props":5642,"children":5645},{"className":5643,"style":5644},[242],"height:0.7667em;vertical-align:-0.0833em;",[],{"type":150,"tag":169,"props":5647,"children":5649},{"className":5648},[248,489],[5650],{"type":155,"value":5559},{"type":150,"tag":169,"props":5652,"children":5654},{"className":5653,"style":5167},[558],[],{"type":150,"tag":169,"props":5656,"children":5658},{"className":5657},[5172],[5659],{"type":155,"value":5554},{"type":150,"tag":169,"props":5661,"children":5663},{"className":5662,"style":5167},[558],[],{"type":150,"tag":169,"props":5665,"children":5667},{"className":5666},[237],[5668,5673,5678,5682,5687],{"type":150,"tag":169,"props":5669,"children":5672},{"className":5670,"style":5671},[242],"height:0.7778em;vertical-align:-0.0833em;",[],{"type":150,"tag":169,"props":5674,"children":5676},{"className":5675},[248,489],[5677],{"type":155,"value":926},{"type":150,"tag":169,"props":5679,"children":5681},{"className":5680,"style":5167},[558],[],{"type":150,"tag":169,"props":5683,"children":5685},{"className":5684},[5172],[5686],{"type":155,"value":5554},{"type":150,"tag":169,"props":5688,"children":5690},{"className":5689,"style":5167},[558],[],{"type":150,"tag":169,"props":5692,"children":5694},{"className":5693},[237],[5695,5699,5756,5760,5765],{"type":150,"tag":169,"props":5696,"children":5698},{"className":5697,"style":4935},[242],[],{"type":150,"tag":169,"props":5700,"children":5702},{"className":5701},[248],[5703,5708],{"type":150,"tag":169,"props":5704,"children":5706},{"className":5705},[248,489],[5707],{"type":155,"value":759},{"type":150,"tag":169,"props":5709,"children":5711},{"className":5710},[495],[5712],{"type":150,"tag":169,"props":5713,"children":5715},{"className":5714},[253,254],[5716,5745],{"type":150,"tag":169,"props":5717,"children":5719},{"className":5718},[259],[5720,5740],{"type":150,"tag":169,"props":5721,"children":5723},{"className":5722,"style":674},[264],[5724],{"type":150,"tag":169,"props":5725,"children":5726},{"style":512},[5727,5731],{"type":150,"tag":169,"props":5728,"children":5730},{"className":5729,"style":517},[274],[],{"type":150,"tag":169,"props":5732,"children":5734},{"className":5733},[522,523,524,525],[5735],{"type":150,"tag":169,"props":5736,"children":5738},{"className":5737,"style":2427},[248,489,525],[5739],{"type":155,"value":1876},{"type":150,"tag":169,"props":5741,"children":5743},{"className":5742},[355],[5744],{"type":155,"value":358},{"type":150,"tag":169,"props":5746,"children":5748},{"className":5747},[259],[5749],{"type":150,"tag":169,"props":5750,"children":5752},{"className":5751,"style":544},[264],[5753],{"type":150,"tag":169,"props":5754,"children":5755},{},[],{"type":150,"tag":169,"props":5757,"children":5759},{"className":5758,"style":5167},[558],[],{"type":150,"tag":169,"props":5761,"children":5763},{"className":5762},[5172],[5764],{"type":155,"value":5554},{"type":150,"tag":169,"props":5766,"children":5768},{"className":5767,"style":5167},[558],[],{"type":150,"tag":169,"props":5770,"children":5772},{"className":5771},[237],[5773,5777,5783,5787,5792],{"type":150,"tag":169,"props":5774,"children":5776},{"className":5775,"style":5644},[242],[],{"type":150,"tag":169,"props":5778,"children":5781},{"className":5779,"style":5780},[248,489],"margin-right:0.109em;",[5782],{"type":155,"value":5591},{"type":150,"tag":169,"props":5784,"children":5786},{"className":5785,"style":5167},[558],[],{"type":150,"tag":169,"props":5788,"children":5790},{"className":5789},[5172],[5791],{"type":155,"value":5554},{"type":150,"tag":169,"props":5793,"children":5795},{"className":5794,"style":5167},[558],[],{"type":150,"tag":169,"props":5797,"children":5799},{"className":5798},[237],[5800,5804],{"type":150,"tag":169,"props":5801,"children":5803},{"className":5802,"style":1121},[242],[],{"type":150,"tag":169,"props":5805,"children":5807},{"className":5806},[248,155],[5808],{"type":150,"tag":169,"props":5809,"children":5811},{"className":5810},[248],[5812],{"type":155,"value":5600},{"type":150,"tag":151,"props":5814,"children":5815},{},[5816,5818,5861,5863,5905,5907,5949,5951,5993],{"type":155,"value":5817},"其中 ",{"type":150,"tag":169,"props":5819,"children":5821},{"className":5820},[177],[5822,5843],{"type":150,"tag":169,"props":5823,"children":5825},{"className":5824},[182],[5826],{"type":150,"tag":185,"props":5827,"children":5828},{"xmlns":187},[5829],{"type":150,"tag":191,"props":5830,"children":5831},{},[5832,5839],{"type":150,"tag":195,"props":5833,"children":5834},{},[5835],{"type":150,"tag":405,"props":5836,"children":5837},{},[5838],{"type":155,"value":5559},{"type":150,"tag":223,"props":5840,"children":5841},{"encoding":225},[5842],{"type":155,"value":5559},{"type":150,"tag":169,"props":5844,"children":5846},{"className":5845,"ariaHidden":214},[232],[5847],{"type":150,"tag":169,"props":5848,"children":5850},{"className":5849},[237],[5851,5856],{"type":150,"tag":169,"props":5852,"children":5855},{"className":5853,"style":5854},[242],"height:0.6833em;",[],{"type":150,"tag":169,"props":5857,"children":5859},{"className":5858},[248,489],[5860],{"type":155,"value":5559},{"type":155,"value":5862}," 是层数，",{"type":150,"tag":169,"props":5864,"children":5866},{"className":5865},[177],[5867,5888],{"type":150,"tag":169,"props":5868,"children":5870},{"className":5869},[182],[5871],{"type":150,"tag":185,"props":5872,"children":5873},{"xmlns":187},[5874],{"type":150,"tag":191,"props":5875,"children":5876},{},[5877,5884],{"type":150,"tag":195,"props":5878,"children":5879},{},[5880],{"type":150,"tag":405,"props":5881,"children":5882},{},[5883],{"type":155,"value":5591},{"type":150,"tag":223,"props":5885,"children":5886},{"encoding":225},[5887],{"type":155,"value":5591},{"type":150,"tag":169,"props":5889,"children":5891},{"className":5890,"ariaHidden":214},[232],[5892],{"type":150,"tag":169,"props":5893,"children":5895},{"className":5894},[237],[5896,5900],{"type":150,"tag":169,"props":5897,"children":5899},{"className":5898,"style":5854},[242],[],{"type":150,"tag":169,"props":5901,"children":5903},{"className":5902,"style":5780},[248,489],[5904],{"type":155,"value":5591},{"type":155,"value":5906}," 是序列长度。注意这里面的 ",{"type":150,"tag":169,"props":5908,"children":5910},{"className":5909},[177],[5911,5932],{"type":150,"tag":169,"props":5912,"children":5914},{"className":5913},[182],[5915],{"type":150,"tag":185,"props":5916,"children":5917},{"xmlns":187},[5918],{"type":150,"tag":191,"props":5919,"children":5920},{},[5921,5928],{"type":150,"tag":195,"props":5922,"children":5923},{},[5924],{"type":150,"tag":405,"props":5925,"children":5926},{},[5927],{"type":155,"value":5591},{"type":150,"tag":223,"props":5929,"children":5930},{"encoding":225},[5931],{"type":155,"value":5591},{"type":150,"tag":169,"props":5933,"children":5935},{"className":5934,"ariaHidden":214},[232],[5936],{"type":150,"tag":169,"props":5937,"children":5939},{"className":5938},[237],[5940,5944],{"type":150,"tag":169,"props":5941,"children":5943},{"className":5942,"style":5854},[242],[],{"type":150,"tag":169,"props":5945,"children":5947},{"className":5946,"style":5780},[248,489],[5948],{"type":155,"value":5591},{"type":155,"value":5950},"。模型参数是个常数，选定模型就不再变了；但 KV Cache 随 ",{"type":150,"tag":169,"props":5952,"children":5954},{"className":5953},[177],[5955,5976],{"type":150,"tag":169,"props":5956,"children":5958},{"className":5957},[182],[5959],{"type":150,"tag":185,"props":5960,"children":5961},{"xmlns":187},[5962],{"type":150,"tag":191,"props":5963,"children":5964},{},[5965,5972],{"type":150,"tag":195,"props":5966,"children":5967},{},[5968],{"type":150,"tag":405,"props":5969,"children":5970},{},[5971],{"type":155,"value":5591},{"type":150,"tag":223,"props":5973,"children":5974},{"encoding":225},[5975],{"type":155,"value":5591},{"type":150,"tag":169,"props":5977,"children":5979},{"className":5978,"ariaHidden":214},[232],[5980],{"type":150,"tag":169,"props":5981,"children":5983},{"className":5982},[237],[5984,5988],{"type":150,"tag":169,"props":5985,"children":5987},{"className":5986,"style":5854},[242],[],{"type":150,"tag":169,"props":5989,"children":5991},{"className":5990,"style":5780},[248,489],[5992],{"type":155,"value":5591},{"type":155,"value":5994}," 线性增长。Context 足够长的时候，它会变成显存里的主导项，可能超出一张卡甚至一台机（8 张卡）的总显存。举个数。按 GPT-3 的配置算：96 层，96 个 attention head，每个 head 维度 128。单个 token 的 KV Cache 是",{"type":150,"tag":169,"props":5996,"children":5998},{"className":5997},[172],[5999],{"type":150,"tag":169,"props":6000,"children":6002},{"className":6001},[177],[6003,6066],{"type":150,"tag":169,"props":6004,"children":6006},{"className":6005},[182],[6007],{"type":150,"tag":185,"props":6008,"children":6009},{"xmlns":187,"display":188},[6010],{"type":150,"tag":191,"props":6011,"children":6012},{},[6013,6061],{"type":150,"tag":195,"props":6014,"children":6015},{},[6016,6020,6024,6029,6033,6037,6041,6046,6051,6056],{"type":150,"tag":411,"props":6017,"children":6018},{},[6019],{"type":155,"value":433},{"type":150,"tag":417,"props":6021,"children":6022},{},[6023],{"type":155,"value":5554},{"type":150,"tag":411,"props":6025,"children":6026},{},[6027],{"type":155,"value":6028},"96",{"type":150,"tag":417,"props":6030,"children":6031},{},[6032],{"type":155,"value":5554},{"type":150,"tag":411,"props":6034,"children":6035},{},[6036],{"type":155,"value":6028},{"type":150,"tag":417,"props":6038,"children":6039},{},[6040],{"type":155,"value":5554},{"type":150,"tag":411,"props":6042,"children":6043},{},[6044],{"type":155,"value":6045},"128",{"type":150,"tag":417,"props":6047,"children":6048},{},[6049],{"type":155,"value":6050},"≈",{"type":150,"tag":411,"props":6052,"children":6053},{},[6054],{"type":155,"value":6055},"2.36",{"type":150,"tag":217,"props":6057,"children":6058},{},[6059],{"type":155,"value":6060},"M",{"type":150,"tag":223,"props":6062,"children":6063},{"encoding":225},[6064],{"type":155,"value":6065},"2 \\times 96 \\times 96 \\times 128 \\approx 2.36\\text{M}",{"type":150,"tag":169,"props":6067,"children":6069},{"className":6068,"ariaHidden":214},[232],[6070,6096,6122,6148,6174],{"type":150,"tag":169,"props":6071,"children":6073},{"className":6072},[237],[6074,6078,6083,6087,6092],{"type":150,"tag":169,"props":6075,"children":6077},{"className":6076,"style":5617},[242],[],{"type":150,"tag":169,"props":6079,"children":6081},{"className":6080},[248],[6082],{"type":155,"value":433},{"type":150,"tag":169,"props":6084,"children":6086},{"className":6085,"style":5167},[558],[],{"type":150,"tag":169,"props":6088,"children":6090},{"className":6089},[5172],[6091],{"type":155,"value":5554},{"type":150,"tag":169,"props":6093,"children":6095},{"className":6094,"style":5167},[558],[],{"type":150,"tag":169,"props":6097,"children":6099},{"className":6098},[237],[6100,6104,6109,6113,6118],{"type":150,"tag":169,"props":6101,"children":6103},{"className":6102,"style":5617},[242],[],{"type":150,"tag":169,"props":6105,"children":6107},{"className":6106},[248],[6108],{"type":155,"value":6028},{"type":150,"tag":169,"props":6110,"children":6112},{"className":6111,"style":5167},[558],[],{"type":150,"tag":169,"props":6114,"children":6116},{"className":6115},[5172],[6117],{"type":155,"value":5554},{"type":150,"tag":169,"props":6119,"children":6121},{"className":6120,"style":5167},[558],[],{"type":150,"tag":169,"props":6123,"children":6125},{"className":6124},[237],[6126,6130,6135,6139,6144],{"type":150,"tag":169,"props":6127,"children":6129},{"className":6128,"style":5617},[242],[],{"type":150,"tag":169,"props":6131,"children":6133},{"className":6132},[248],[6134],{"type":155,"value":6028},{"type":150,"tag":169,"props":6136,"children":6138},{"className":6137,"style":5167},[558],[],{"type":150,"tag":169,"props":6140,"children":6142},{"className":6141},[5172],[6143],{"type":155,"value":5554},{"type":150,"tag":169,"props":6145,"children":6147},{"className":6146,"style":5167},[558],[],{"type":150,"tag":169,"props":6149,"children":6151},{"className":6150},[237],[6152,6156,6161,6165,6170],{"type":150,"tag":169,"props":6153,"children":6155},{"className":6154,"style":5186},[242],[],{"type":150,"tag":169,"props":6157,"children":6159},{"className":6158},[248],[6160],{"type":155,"value":6045},{"type":150,"tag":169,"props":6162,"children":6164},{"className":6163,"style":839},[558],[],{"type":150,"tag":169,"props":6166,"children":6168},{"className":6167},[844],[6169],{"type":155,"value":6050},{"type":150,"tag":169,"props":6171,"children":6173},{"className":6172,"style":839},[558],[],{"type":150,"tag":169,"props":6175,"children":6177},{"className":6176},[237],[6178,6182,6187],{"type":150,"tag":169,"props":6179,"children":6181},{"className":6180,"style":5854},[242],[],{"type":150,"tag":169,"props":6183,"children":6185},{"className":6184},[248],[6186],{"type":155,"value":6055},{"type":150,"tag":169,"props":6188,"children":6190},{"className":6189},[248,155],[6191],{"type":150,"tag":169,"props":6192,"children":6194},{"className":6193},[248],[6195],{"type":155,"value":6060},{"type":150,"tag":151,"props":6197,"children":6198},{},[6199],{"type":155,"value":6200},"个参数。按 2 字节精度存，就是 4.7 MB。一个 token 4.7 MB。128k context 就是 600 GB 量级。这个数字基本说明了为什么长上下文不是\"再多加个位置编码\"就能解决的。它是个显存和带宽问题。而 GPU 部署有个很不讲道理的原则：能一张卡放下的，就不要跨多张卡；能一台机放下的，就不要跨多台机。因为卡内通信带宽 > 卡间通信带宽 > 机间通信带宽。木桶效应，跨的设备越多，被最慢那段拖累得越狠。",{"type":150,"tag":151,"props":6202,"children":6203},{},[6204],{"type":155,"value":6205},"单卡 H100 内 SRAM 到 HBM 的带宽已经有 3 TB\u002Fs，对 short context 来说这依然是瓶颈，更不用说更慢的卡间和机间了。",{"type":150,"tag":372,"props":6207,"children":6209},{"id":6208},"prefill-与-decode",[6210],{"type":155,"value":6211},"Prefill 与 Decode",{"type":150,"tag":151,"props":6213,"children":6214},{},[6215,6217,6222,6224,6229,6230,6235],{"type":155,"value":6216},"还有一件事必须拆开说，因为后面 MLA 的设计直接从这里来。LLM 推理分两个阶段，它们的成本结构完全不是一回事。",{"type":150,"tag":158,"props":6218,"children":6219},{},[6220],{"type":155,"value":6221},"Prefill",{"type":155,"value":6223},"：把输入的所有 token 并行算一遍，同时把 KV Cache 建起来。所有 token 一起算，矩阵运算排得很满，算力基本能吃满。它是 ",{"type":150,"tag":158,"props":6225,"children":6226},{},[6227],{"type":155,"value":6228},"compute bound",{"type":155,"value":904},{"type":150,"tag":158,"props":6231,"children":6232},{},[6233],{"type":155,"value":6234},"Decode",{"type":155,"value":6236},"：之后逐个生成 token。这里有个容易漏掉的点。",{"type":150,"tag":151,"props":6238,"children":6239},{},[6240,6242,6247],{"type":155,"value":6241},"每生成一个 token，计算量很小，但要读的东西一点都不少：",{"type":150,"tag":158,"props":6243,"children":6244},{},[6245],{"type":155,"value":6246},"模型权重",{"type":155,"value":6248},"要整体过一遍，KV Cache 也要读一遍。",{"type":150,"tag":151,"props":6250,"children":6251},{},[6252,6254,6259],{"type":155,"value":6253},"不是读一部分权重，也不是只读 attention 那几层。每一步都要把参与计算的权重从 HBM 搬进 SM。算得少，搬得多。所以 decode 是 ",{"type":150,"tag":158,"props":6255,"children":6256},{},[6257],{"type":155,"value":6258},"memory bound",{"type":155,"value":6260},"。同一个模型，两个阶段的瓶颈完全不同。一个是 compute bound，一个是 memory bound。这件事的重要性在于：如果一个架构在两种形式下数学等价，那就可以 prefill 用一种算，decode 用另一种算，各取所需。MLA 就是这么干的。",{"type":150,"tag":151,"props":6262,"children":6263},{},[6264],{"type":155,"value":6265},"后面 V4.1 那一节会把这两个瓶颈各自的账算一遍。最后把账单写全：",{"type":150,"tag":169,"props":6267,"children":6269},{"className":6268},[172],[6270],{"type":150,"tag":169,"props":6271,"children":6273},{"className":6272},[177],[6274,6353],{"type":150,"tag":169,"props":6275,"children":6277},{"className":6276},[182],[6278],{"type":150,"tag":185,"props":6279,"children":6280},{"xmlns":187,"display":188},[6281],{"type":150,"tag":191,"props":6282,"children":6283},{},[6284,6348],{"type":150,"tag":195,"props":6285,"children":6286},{},[6287,6300,6304,6316,6320,6332,6336],{"type":150,"tag":401,"props":6288,"children":6289},{},[6290,6295],{"type":150,"tag":405,"props":6291,"children":6292},{},[6293],{"type":155,"value":6294},"C",{"type":150,"tag":217,"props":6296,"children":6297},{},[6298],{"type":155,"value":6299},"total",{"type":150,"tag":417,"props":6301,"children":6302},{},[6303],{"type":155,"value":1045},{"type":150,"tag":401,"props":6305,"children":6306},{},[6307,6311],{"type":150,"tag":405,"props":6308,"children":6309},{},[6310],{"type":155,"value":6294},{"type":150,"tag":217,"props":6312,"children":6313},{},[6314],{"type":155,"value":6315},"attention",{"type":150,"tag":417,"props":6317,"children":6318},{},[6319],{"type":155,"value":5136},{"type":150,"tag":401,"props":6321,"children":6322},{},[6323,6327],{"type":150,"tag":405,"props":6324,"children":6325},{},[6326],{"type":155,"value":6294},{"type":150,"tag":217,"props":6328,"children":6329},{},[6330],{"type":155,"value":6331},"FFN\u002FMoE",{"type":150,"tag":417,"props":6333,"children":6334},{},[6335],{"type":155,"value":5136},{"type":150,"tag":401,"props":6337,"children":6338},{},[6339,6343],{"type":150,"tag":405,"props":6340,"children":6341},{},[6342],{"type":155,"value":6294},{"type":150,"tag":217,"props":6344,"children":6345},{},[6346],{"type":155,"value":6347},"memory movement",{"type":150,"tag":223,"props":6349,"children":6350},{"encoding":225},[6351],{"type":155,"value":6352},"C_{\\text{total}} = C_{\\text{attention}} + C_{\\text{FFN\u002FMoE}} + C_{\\text{memory movement}}",{"type":150,"tag":169,"props":6354,"children":6356},{"className":6355,"ariaHidden":214},[232],[6357,6446,6533,6622],{"type":150,"tag":169,"props":6358,"children":6360},{"className":6359},[237],[6361,6366,6433,6437,6442],{"type":150,"tag":169,"props":6362,"children":6365},{"className":6363,"style":6364},[242],"height:0.8333em;vertical-align:-0.15em;",[],{"type":150,"tag":169,"props":6367,"children":6369},{"className":6368},[248],[6370,6376],{"type":150,"tag":169,"props":6371,"children":6374},{"className":6372,"style":6373},[248,489],"margin-right:0.0715em;",[6375],{"type":155,"value":6294},{"type":150,"tag":169,"props":6377,"children":6379},{"className":6378},[495],[6380],{"type":150,"tag":169,"props":6381,"children":6383},{"className":6382},[253,254],[6384,6422],{"type":150,"tag":169,"props":6385,"children":6387},{"className":6386},[259],[6388,6417],{"type":150,"tag":169,"props":6389,"children":6391},{"className":6390,"style":674},[264],[6392],{"type":150,"tag":169,"props":6393,"children":6395},{"style":6394},"top:-2.55em;margin-left:-0.0715em;margin-right:0.05em;",[6396,6400],{"type":150,"tag":169,"props":6397,"children":6399},{"className":6398,"style":517},[274],[],{"type":150,"tag":169,"props":6401,"children":6403},{"className":6402},[522,523,524,525],[6404],{"type":150,"tag":169,"props":6405,"children":6407},{"className":6406},[248,525],[6408],{"type":150,"tag":169,"props":6409,"children":6411},{"className":6410},[248,155,525],[6412],{"type":150,"tag":169,"props":6413,"children":6415},{"className":6414},[248,525],[6416],{"type":155,"value":6299},{"type":150,"tag":169,"props":6418,"children":6420},{"className":6419},[355],[6421],{"type":155,"value":358},{"type":150,"tag":169,"props":6423,"children":6425},{"className":6424},[259],[6426],{"type":150,"tag":169,"props":6427,"children":6429},{"className":6428,"style":544},[264],[6430],{"type":150,"tag":169,"props":6431,"children":6432},{},[],{"type":150,"tag":169,"props":6434,"children":6436},{"className":6435,"style":839},[558],[],{"type":150,"tag":169,"props":6438,"children":6440},{"className":6439},[844],[6441],{"type":155,"value":1045},{"type":150,"tag":169,"props":6443,"children":6445},{"className":6444,"style":839},[558],[],{"type":150,"tag":169,"props":6447,"children":6449},{"className":6448},[237],[6450,6454,6520,6524,6529],{"type":150,"tag":169,"props":6451,"children":6453},{"className":6452,"style":6364},[242],[],{"type":150,"tag":169,"props":6455,"children":6457},{"className":6456},[248],[6458,6463],{"type":150,"tag":169,"props":6459,"children":6461},{"className":6460,"style":6373},[248,489],[6462],{"type":155,"value":6294},{"type":150,"tag":169,"props":6464,"children":6466},{"className":6465},[495],[6467],{"type":150,"tag":169,"props":6468,"children":6470},{"className":6469},[253,254],[6471,6509],{"type":150,"tag":169,"props":6472,"children":6474},{"className":6473},[259],[6475,6504],{"type":150,"tag":169,"props":6476,"children":6479},{"className":6477,"style":6478},[264],"height:0.3175em;",[6480],{"type":150,"tag":169,"props":6481,"children":6482},{"style":6394},[6483,6487],{"type":150,"tag":169,"props":6484,"children":6486},{"className":6485,"style":517},[274],[],{"type":150,"tag":169,"props":6488,"children":6490},{"className":6489},[522,523,524,525],[6491],{"type":150,"tag":169,"props":6492,"children":6494},{"className":6493},[248,525],[6495],{"type":150,"tag":169,"props":6496,"children":6498},{"className":6497},[248,155,525],[6499],{"type":150,"tag":169,"props":6500,"children":6502},{"className":6501},[248,525],[6503],{"type":155,"value":6315},{"type":150,"tag":169,"props":6505,"children":6507},{"className":6506},[355],[6508],{"type":155,"value":358},{"type":150,"tag":169,"props":6510,"children":6512},{"className":6511},[259],[6513],{"type":150,"tag":169,"props":6514,"children":6516},{"className":6515,"style":544},[264],[6517],{"type":150,"tag":169,"props":6518,"children":6519},{},[],{"type":150,"tag":169,"props":6521,"children":6523},{"className":6522,"style":5167},[558],[],{"type":150,"tag":169,"props":6525,"children":6527},{"className":6526},[5172],[6528],{"type":155,"value":5136},{"type":150,"tag":169,"props":6530,"children":6532},{"className":6531,"style":5167},[558],[],{"type":150,"tag":169,"props":6534,"children":6536},{"className":6535},[237],[6537,6542,6609,6613,6618],{"type":150,"tag":169,"props":6538,"children":6541},{"className":6539,"style":6540},[242],"height:1.0385em;vertical-align:-0.3552em;",[],{"type":150,"tag":169,"props":6543,"children":6545},{"className":6544},[248],[6546,6551],{"type":150,"tag":169,"props":6547,"children":6549},{"className":6548,"style":6373},[248,489],[6550],{"type":155,"value":6294},{"type":150,"tag":169,"props":6552,"children":6554},{"className":6553},[495],[6555],{"type":150,"tag":169,"props":6556,"children":6558},{"className":6557},[253,254],[6559,6597],{"type":150,"tag":169,"props":6560,"children":6562},{"className":6561},[259],[6563,6592],{"type":150,"tag":169,"props":6564,"children":6566},{"className":6565,"style":4040},[264],[6567],{"type":150,"tag":169,"props":6568,"children":6570},{"style":6569},"top:-2.5198em;margin-left:-0.0715em;margin-right:0.05em;",[6571,6575],{"type":150,"tag":169,"props":6572,"children":6574},{"className":6573,"style":517},[274],[],{"type":150,"tag":169,"props":6576,"children":6578},{"className":6577},[522,523,524,525],[6579],{"type":150,"tag":169,"props":6580,"children":6582},{"className":6581},[248,525],[6583],{"type":150,"tag":169,"props":6584,"children":6586},{"className":6585},[248,155,525],[6587],{"type":150,"tag":169,"props":6588,"children":6590},{"className":6589},[248,525],[6591],{"type":155,"value":6331},{"type":150,"tag":169,"props":6593,"children":6595},{"className":6594},[355],[6596],{"type":155,"value":358},{"type":150,"tag":169,"props":6598,"children":6600},{"className":6599},[259],[6601],{"type":150,"tag":169,"props":6602,"children":6605},{"className":6603,"style":6604},[264],"height:0.3552em;",[6606],{"type":150,"tag":169,"props":6607,"children":6608},{},[],{"type":150,"tag":169,"props":6610,"children":6612},{"className":6611,"style":5167},[558],[],{"type":150,"tag":169,"props":6614,"children":6616},{"className":6615},[5172],[6617],{"type":155,"value":5136},{"type":150,"tag":169,"props":6619,"children":6621},{"className":6620,"style":5167},[558],[],{"type":150,"tag":169,"props":6623,"children":6625},{"className":6624},[237],[6626,6631],{"type":150,"tag":169,"props":6627,"children":6630},{"className":6628,"style":6629},[242],"height:0.9694em;vertical-align:-0.2861em;",[],{"type":150,"tag":169,"props":6632,"children":6634},{"className":6633},[248],[6635,6640],{"type":150,"tag":169,"props":6636,"children":6638},{"className":6637,"style":6373},[248,489],[6639],{"type":155,"value":6294},{"type":150,"tag":169,"props":6641,"children":6643},{"className":6642},[495],[6644],{"type":150,"tag":169,"props":6645,"children":6647},{"className":6646},[253,254],[6648,6685],{"type":150,"tag":169,"props":6649,"children":6651},{"className":6650},[259],[6652,6680],{"type":150,"tag":169,"props":6653,"children":6655},{"className":6654,"style":1369},[264],[6656],{"type":150,"tag":169,"props":6657,"children":6658},{"style":6394},[6659,6663],{"type":150,"tag":169,"props":6660,"children":6662},{"className":6661,"style":517},[274],[],{"type":150,"tag":169,"props":6664,"children":6666},{"className":6665},[522,523,524,525],[6667],{"type":150,"tag":169,"props":6668,"children":6670},{"className":6669},[248,525],[6671],{"type":150,"tag":169,"props":6672,"children":6674},{"className":6673},[248,155,525],[6675],{"type":150,"tag":169,"props":6676,"children":6678},{"className":6677},[248,525],[6679],{"type":155,"value":6347},{"type":150,"tag":169,"props":6681,"children":6683},{"className":6682},[355],[6684],{"type":155,"value":358},{"type":150,"tag":169,"props":6686,"children":6688},{"className":6687},[259],[6689],{"type":150,"tag":169,"props":6690,"children":6693},{"className":6691,"style":6692},[264],"height:0.2861em;",[6694],{"type":150,"tag":169,"props":6695,"children":6696},{},[],{"type":150,"tag":151,"props":6698,"children":6699},{},[6700,6702,6812,6814,6856,6858,6900,6902,7028,7030,7152],{"type":155,"value":6701},"账单上还有另外两项，而且它们经常被无视。第二项 ",{"type":150,"tag":169,"props":6703,"children":6705},{"className":6704},[177],[6706,6735],{"type":150,"tag":169,"props":6707,"children":6709},{"className":6708},[182],[6710],{"type":150,"tag":185,"props":6711,"children":6712},{"xmlns":187},[6713],{"type":150,"tag":191,"props":6714,"children":6715},{},[6716,6730],{"type":150,"tag":195,"props":6717,"children":6718},{},[6719],{"type":150,"tag":401,"props":6720,"children":6721},{},[6722,6726],{"type":150,"tag":405,"props":6723,"children":6724},{},[6725],{"type":155,"value":6294},{"type":150,"tag":217,"props":6727,"children":6728},{},[6729],{"type":155,"value":6331},{"type":150,"tag":223,"props":6731,"children":6732},{"encoding":225},[6733],{"type":155,"value":6734},"C_{\\text{FFN\u002FMoE}}",{"type":150,"tag":169,"props":6736,"children":6738},{"className":6737,"ariaHidden":214},[232],[6739],{"type":150,"tag":169,"props":6740,"children":6742},{"className":6741},[237],[6743,6747],{"type":150,"tag":169,"props":6744,"children":6746},{"className":6745,"style":6540},[242],[],{"type":150,"tag":169,"props":6748,"children":6750},{"className":6749},[248],[6751,6756],{"type":150,"tag":169,"props":6752,"children":6754},{"className":6753,"style":6373},[248,489],[6755],{"type":155,"value":6294},{"type":150,"tag":169,"props":6757,"children":6759},{"className":6758},[495],[6760],{"type":150,"tag":169,"props":6761,"children":6763},{"className":6762},[253,254],[6764,6801],{"type":150,"tag":169,"props":6765,"children":6767},{"className":6766},[259],[6768,6796],{"type":150,"tag":169,"props":6769,"children":6771},{"className":6770,"style":4040},[264],[6772],{"type":150,"tag":169,"props":6773,"children":6774},{"style":6569},[6775,6779],{"type":150,"tag":169,"props":6776,"children":6778},{"className":6777,"style":517},[274],[],{"type":150,"tag":169,"props":6780,"children":6782},{"className":6781},[522,523,524,525],[6783],{"type":150,"tag":169,"props":6784,"children":6786},{"className":6785},[248,525],[6787],{"type":150,"tag":169,"props":6788,"children":6790},{"className":6789},[248,155,525],[6791],{"type":150,"tag":169,"props":6792,"children":6794},{"className":6793},[248,525],[6795],{"type":155,"value":6331},{"type":150,"tag":169,"props":6797,"children":6799},{"className":6798},[355],[6800],{"type":155,"value":358},{"type":150,"tag":169,"props":6802,"children":6804},{"className":6803},[259],[6805],{"type":150,"tag":169,"props":6806,"children":6808},{"className":6807,"style":6604},[264],[6809],{"type":150,"tag":169,"props":6810,"children":6811},{},[],{"type":155,"value":6813}," 是前馈层的计算。MoE（Mixture of Experts）就是把这层换成多个专家网络，每个 token 只激活其中少数几个。它跟序列长度 ",{"type":150,"tag":169,"props":6815,"children":6817},{"className":6816},[177],[6818,6839],{"type":150,"tag":169,"props":6819,"children":6821},{"className":6820},[182],[6822],{"type":150,"tag":185,"props":6823,"children":6824},{"xmlns":187},[6825],{"type":150,"tag":191,"props":6826,"children":6827},{},[6828,6835],{"type":150,"tag":195,"props":6829,"children":6830},{},[6831],{"type":150,"tag":405,"props":6832,"children":6833},{},[6834],{"type":155,"value":5591},{"type":150,"tag":223,"props":6836,"children":6837},{"encoding":225},[6838],{"type":155,"value":5591},{"type":150,"tag":169,"props":6840,"children":6842},{"className":6841,"ariaHidden":214},[232],[6843],{"type":150,"tag":169,"props":6844,"children":6846},{"className":6845},[237],[6847,6851],{"type":150,"tag":169,"props":6848,"children":6850},{"className":6849,"style":5854},[242],[],{"type":150,"tag":169,"props":6852,"children":6854},{"className":6853,"style":5780},[248,489],[6855],{"type":155,"value":5591},{"type":155,"value":6857}," 成正比，跟模型宽度 ",{"type":150,"tag":169,"props":6859,"children":6861},{"className":6860},[177],[6862,6883],{"type":150,"tag":169,"props":6863,"children":6865},{"className":6864},[182],[6866],{"type":150,"tag":185,"props":6867,"children":6868},{"xmlns":187},[6869],{"type":150,"tag":191,"props":6870,"children":6871},{},[6872,6879],{"type":150,"tag":195,"props":6873,"children":6874},{},[6875],{"type":150,"tag":405,"props":6876,"children":6877},{},[6878],{"type":155,"value":759},{"type":150,"tag":223,"props":6880,"children":6881},{"encoding":225},[6882],{"type":155,"value":759},{"type":150,"tag":169,"props":6884,"children":6886},{"className":6885,"ariaHidden":214},[232],[6887],{"type":150,"tag":169,"props":6888,"children":6890},{"className":6889},[237],[6891,6895],{"type":150,"tag":169,"props":6892,"children":6894},{"className":6893,"style":942},[242],[],{"type":150,"tag":169,"props":6896,"children":6898},{"className":6897},[248,489],[6899],{"type":155,"value":759},{"type":155,"value":6901}," 的平方成正比，所以量级是 ",{"type":150,"tag":169,"props":6903,"children":6905},{"className":6904},[177],[6906,6952],{"type":150,"tag":169,"props":6907,"children":6909},{"className":6908},[182],[6910],{"type":150,"tag":185,"props":6911,"children":6912},{"xmlns":187},[6913],{"type":150,"tag":191,"props":6914,"children":6915},{},[6916,6947],{"type":150,"tag":195,"props":6917,"children":6918},{},[6919,6924,6928,6932,6943],{"type":150,"tag":405,"props":6920,"children":6921},{},[6922],{"type":155,"value":6923},"O",{"type":150,"tag":417,"props":6925,"children":6926},{"stretchy":207},[6927],{"type":155,"value":972},{"type":150,"tag":405,"props":6929,"children":6930},{},[6931],{"type":155,"value":5591},{"type":150,"tag":746,"props":6933,"children":6934},{},[6935,6939],{"type":150,"tag":405,"props":6936,"children":6937},{},[6938],{"type":155,"value":759},{"type":150,"tag":411,"props":6940,"children":6941},{},[6942],{"type":155,"value":433},{"type":150,"tag":417,"props":6944,"children":6945},{"stretchy":207},[6946],{"type":155,"value":982},{"type":150,"tag":223,"props":6948,"children":6949},{"encoding":225},[6950],{"type":155,"value":6951},"O(Nd^2)",{"type":150,"tag":169,"props":6953,"children":6955},{"className":6954,"ariaHidden":214},[232],[6956],{"type":150,"tag":169,"props":6957,"children":6959},{"className":6958},[237],[6960,6965,6971,6976,6981,7023],{"type":150,"tag":169,"props":6961,"children":6964},{"className":6962,"style":6963},[242],"height:1.0641em;vertical-align:-0.25em;",[],{"type":150,"tag":169,"props":6966,"children":6969},{"className":6967,"style":6968},[248,489],"margin-right:0.0278em;",[6970],{"type":155,"value":6923},{"type":150,"tag":169,"props":6972,"children":6974},{"className":6973},[1004],[6975],{"type":155,"value":972},{"type":150,"tag":169,"props":6977,"children":6979},{"className":6978,"style":5780},[248,489],[6980],{"type":155,"value":5591},{"type":150,"tag":169,"props":6982,"children":6984},{"className":6983},[248],[6985,6990],{"type":150,"tag":169,"props":6986,"children":6988},{"className":6987},[248,489],[6989],{"type":155,"value":759},{"type":150,"tag":169,"props":6991,"children":6993},{"className":6992},[495],[6994],{"type":150,"tag":169,"props":6995,"children":6997},{"className":6996},[253],[6998],{"type":150,"tag":169,"props":6999,"children":7001},{"className":7000},[259],[7002],{"type":150,"tag":169,"props":7003,"children":7006},{"className":7004,"style":7005},[264],"height:0.8141em;",[7007],{"type":150,"tag":169,"props":7008,"children":7009},{"style":888},[7010,7014],{"type":150,"tag":169,"props":7011,"children":7013},{"className":7012,"style":517},[274],[],{"type":150,"tag":169,"props":7015,"children":7017},{"className":7016},[522,523,524,525],[7018],{"type":150,"tag":169,"props":7019,"children":7021},{"className":7020},[248,525],[7022],{"type":155,"value":433},{"type":150,"tag":169,"props":7024,"children":7026},{"className":7025},[1015],[7027],{"type":155,"value":982},{"type":155,"value":7029},"。attention 那项是 ",{"type":150,"tag":169,"props":7031,"children":7033},{"className":7032},[177],[7034,7079],{"type":150,"tag":169,"props":7035,"children":7037},{"className":7036},[182],[7038],{"type":150,"tag":185,"props":7039,"children":7040},{"xmlns":187},[7041],{"type":150,"tag":191,"props":7042,"children":7043},{},[7044,7074],{"type":150,"tag":195,"props":7045,"children":7046},{},[7047,7051,7055,7066,7070],{"type":150,"tag":405,"props":7048,"children":7049},{},[7050],{"type":155,"value":6923},{"type":150,"tag":417,"props":7052,"children":7053},{"stretchy":207},[7054],{"type":155,"value":972},{"type":150,"tag":746,"props":7056,"children":7057},{},[7058,7062],{"type":150,"tag":405,"props":7059,"children":7060},{},[7061],{"type":155,"value":5591},{"type":150,"tag":411,"props":7063,"children":7064},{},[7065],{"type":155,"value":433},{"type":150,"tag":405,"props":7067,"children":7068},{},[7069],{"type":155,"value":759},{"type":150,"tag":417,"props":7071,"children":7072},{"stretchy":207},[7073],{"type":155,"value":982},{"type":150,"tag":223,"props":7075,"children":7076},{"encoding":225},[7077],{"type":155,"value":7078},"O(N^2 d)",{"type":150,"tag":169,"props":7080,"children":7082},{"className":7081,"ariaHidden":214},[232],[7083],{"type":150,"tag":169,"props":7084,"children":7086},{"className":7085},[237],[7087,7091,7096,7101,7142,7147],{"type":150,"tag":169,"props":7088,"children":7090},{"className":7089,"style":6963},[242],[],{"type":150,"tag":169,"props":7092,"children":7094},{"className":7093,"style":6968},[248,489],[7095],{"type":155,"value":6923},{"type":150,"tag":169,"props":7097,"children":7099},{"className":7098},[1004],[7100],{"type":155,"value":972},{"type":150,"tag":169,"props":7102,"children":7104},{"className":7103},[248],[7105,7110],{"type":150,"tag":169,"props":7106,"children":7108},{"className":7107,"style":5780},[248,489],[7109],{"type":155,"value":5591},{"type":150,"tag":169,"props":7111,"children":7113},{"className":7112},[495],[7114],{"type":150,"tag":169,"props":7115,"children":7117},{"className":7116},[253],[7118],{"type":150,"tag":169,"props":7119,"children":7121},{"className":7120},[259],[7122],{"type":150,"tag":169,"props":7123,"children":7125},{"className":7124,"style":7005},[264],[7126],{"type":150,"tag":169,"props":7127,"children":7128},{"style":888},[7129,7133],{"type":150,"tag":169,"props":7130,"children":7132},{"className":7131,"style":517},[274],[],{"type":150,"tag":169,"props":7134,"children":7136},{"className":7135},[522,523,524,525],[7137],{"type":150,"tag":169,"props":7138,"children":7140},{"className":7139},[248,525],[7141],{"type":155,"value":433},{"type":150,"tag":169,"props":7143,"children":7145},{"className":7144},[248,489],[7146],{"type":155,"value":759},{"type":150,"tag":169,"props":7148,"children":7150},{"className":7149},[1015],[7151],{"type":155,"value":982},{"type":155,"value":7153},"。注意这两个量级里的指数位置不一样。",{"type":150,"tag":151,"props":7155,"children":7156},{},[7157,7199,7201,7296,7298,7393,7395,7437],{"type":150,"tag":169,"props":7158,"children":7160},{"className":7159},[177],[7161,7182],{"type":150,"tag":169,"props":7162,"children":7164},{"className":7163},[182],[7165],{"type":150,"tag":185,"props":7166,"children":7167},{"xmlns":187},[7168],{"type":150,"tag":191,"props":7169,"children":7170},{},[7171,7178],{"type":150,"tag":195,"props":7172,"children":7173},{},[7174],{"type":150,"tag":405,"props":7175,"children":7176},{},[7177],{"type":155,"value":5591},{"type":150,"tag":223,"props":7179,"children":7180},{"encoding":225},[7181],{"type":155,"value":5591},{"type":150,"tag":169,"props":7183,"children":7185},{"className":7184,"ariaHidden":214},[232],[7186],{"type":150,"tag":169,"props":7187,"children":7189},{"className":7188},[237],[7190,7194],{"type":150,"tag":169,"props":7191,"children":7193},{"className":7192,"style":5854},[242],[],{"type":150,"tag":169,"props":7195,"children":7197},{"className":7196,"style":5780},[248,489],[7198],{"type":155,"value":5591},{"type":155,"value":7200}," 小的时候，",{"type":150,"tag":169,"props":7202,"children":7204},{"className":7203},[177],[7205,7238],{"type":150,"tag":169,"props":7206,"children":7208},{"className":7207},[182],[7209],{"type":150,"tag":185,"props":7210,"children":7211},{"xmlns":187},[7212],{"type":150,"tag":191,"props":7213,"children":7214},{},[7215,7233],{"type":150,"tag":195,"props":7216,"children":7217},{},[7218,7229],{"type":150,"tag":746,"props":7219,"children":7220},{},[7221,7225],{"type":150,"tag":405,"props":7222,"children":7223},{},[7224],{"type":155,"value":5591},{"type":150,"tag":411,"props":7226,"children":7227},{},[7228],{"type":155,"value":433},{"type":150,"tag":405,"props":7230,"children":7231},{},[7232],{"type":155,"value":759},{"type":150,"tag":223,"props":7234,"children":7235},{"encoding":225},[7236],{"type":155,"value":7237},"N^2 d",{"type":150,"tag":169,"props":7239,"children":7241},{"className":7240,"ariaHidden":214},[232],[7242],{"type":150,"tag":169,"props":7243,"children":7245},{"className":7244},[237],[7246,7250,7291],{"type":150,"tag":169,"props":7247,"children":7249},{"className":7248,"style":7005},[242],[],{"type":150,"tag":169,"props":7251,"children":7253},{"className":7252},[248],[7254,7259],{"type":150,"tag":169,"props":7255,"children":7257},{"className":7256,"style":5780},[248,489],[7258],{"type":155,"value":5591},{"type":150,"tag":169,"props":7260,"children":7262},{"className":7261},[495],[7263],{"type":150,"tag":169,"props":7264,"children":7266},{"className":7265},[253],[7267],{"type":150,"tag":169,"props":7268,"children":7270},{"className":7269},[259],[7271],{"type":150,"tag":169,"props":7272,"children":7274},{"className":7273,"style":7005},[264],[7275],{"type":150,"tag":169,"props":7276,"children":7277},{"style":888},[7278,7282],{"type":150,"tag":169,"props":7279,"children":7281},{"className":7280,"style":517},[274],[],{"type":150,"tag":169,"props":7283,"children":7285},{"className":7284},[522,523,524,525],[7286],{"type":150,"tag":169,"props":7287,"children":7289},{"className":7288},[248,525],[7290],{"type":155,"value":433},{"type":150,"tag":169,"props":7292,"children":7294},{"className":7293},[248,489],[7295],{"type":155,"value":759},{"type":155,"value":7297}," 打不过 ",{"type":150,"tag":169,"props":7299,"children":7301},{"className":7300},[177],[7302,7335],{"type":150,"tag":169,"props":7303,"children":7305},{"className":7304},[182],[7306],{"type":150,"tag":185,"props":7307,"children":7308},{"xmlns":187},[7309],{"type":150,"tag":191,"props":7310,"children":7311},{},[7312,7330],{"type":150,"tag":195,"props":7313,"children":7314},{},[7315,7319],{"type":150,"tag":405,"props":7316,"children":7317},{},[7318],{"type":155,"value":5591},{"type":150,"tag":746,"props":7320,"children":7321},{},[7322,7326],{"type":150,"tag":405,"props":7323,"children":7324},{},[7325],{"type":155,"value":759},{"type":150,"tag":411,"props":7327,"children":7328},{},[7329],{"type":155,"value":433},{"type":150,"tag":223,"props":7331,"children":7332},{"encoding":225},[7333],{"type":155,"value":7334},"Nd^2",{"type":150,"tag":169,"props":7336,"children":7338},{"className":7337,"ariaHidden":214},[232],[7339],{"type":150,"tag":169,"props":7340,"children":7342},{"className":7341},[237],[7343,7347,7352],{"type":150,"tag":169,"props":7344,"children":7346},{"className":7345,"style":7005},[242],[],{"type":150,"tag":169,"props":7348,"children":7350},{"className":7349,"style":5780},[248,489],[7351],{"type":155,"value":5591},{"type":150,"tag":169,"props":7353,"children":7355},{"className":7354},[248],[7356,7361],{"type":150,"tag":169,"props":7357,"children":7359},{"className":7358},[248,489],[7360],{"type":155,"value":759},{"type":150,"tag":169,"props":7362,"children":7364},{"className":7363},[495],[7365],{"type":150,"tag":169,"props":7366,"children":7368},{"className":7367},[253],[7369],{"type":150,"tag":169,"props":7370,"children":7372},{"className":7371},[259],[7373],{"type":150,"tag":169,"props":7374,"children":7376},{"className":7375,"style":7005},[264],[7377],{"type":150,"tag":169,"props":7378,"children":7379},{"style":888},[7380,7384],{"type":150,"tag":169,"props":7381,"children":7383},{"className":7382,"style":517},[274],[],{"type":150,"tag":169,"props":7385,"children":7387},{"className":7386},[522,523,524,525],[7388],{"type":150,"tag":169,"props":7389,"children":7391},{"className":7390},[248,525],[7392],{"type":155,"value":433},{"type":155,"value":7394},"；",{"type":150,"tag":169,"props":7396,"children":7398},{"className":7397},[177],[7399,7420],{"type":150,"tag":169,"props":7400,"children":7402},{"className":7401},[182],[7403],{"type":150,"tag":185,"props":7404,"children":7405},{"xmlns":187},[7406],{"type":150,"tag":191,"props":7407,"children":7408},{},[7409,7416],{"type":150,"tag":195,"props":7410,"children":7411},{},[7412],{"type":150,"tag":405,"props":7413,"children":7414},{},[7415],{"type":155,"value":5591},{"type":150,"tag":223,"props":7417,"children":7418},{"encoding":225},[7419],{"type":155,"value":5591},{"type":150,"tag":169,"props":7421,"children":7423},{"className":7422,"ariaHidden":214},[232],[7424],{"type":150,"tag":169,"props":7425,"children":7427},{"className":7426},[237],[7428,7432],{"type":150,"tag":169,"props":7429,"children":7431},{"className":7430,"style":5854},[242],[],{"type":150,"tag":169,"props":7433,"children":7435},{"className":7434,"style":5780},[248,489],[7436],{"type":155,"value":5591},{"type":155,"value":7438}," 大起来之后才反过来。",{"type":150,"tag":151,"props":7440,"children":7441},{},[7442,7444,7486,7488,7574,7576,7697,7699,7809],{"type":155,"value":7443},"这也是为什么在短 context 时代大家理所当然地只盯 attention。那时候它确实是瓶颈。但当 ",{"type":150,"tag":169,"props":7445,"children":7447},{"className":7446},[177],[7448,7469],{"type":150,"tag":169,"props":7449,"children":7451},{"className":7450},[182],[7452],{"type":150,"tag":185,"props":7453,"children":7454},{"xmlns":187},[7455],{"type":150,"tag":191,"props":7456,"children":7457},{},[7458,7465],{"type":150,"tag":195,"props":7459,"children":7460},{},[7461],{"type":150,"tag":405,"props":7462,"children":7463},{},[7464],{"type":155,"value":5591},{"type":150,"tag":223,"props":7466,"children":7467},{"encoding":225},[7468],{"type":155,"value":5591},{"type":150,"tag":169,"props":7470,"children":7472},{"className":7471,"ariaHidden":214},[232],[7473],{"type":150,"tag":169,"props":7474,"children":7476},{"className":7475},[237],[7477,7481],{"type":150,"tag":169,"props":7478,"children":7480},{"className":7479,"style":5854},[242],[],{"type":150,"tag":169,"props":7482,"children":7484},{"className":7483,"style":5780},[248,489],[7485],{"type":155,"value":5591},{"type":155,"value":7487}," 从 4k 涨到 1M，情况变了；再等到 sparse attention 把 ",{"type":150,"tag":169,"props":7489,"children":7491},{"className":7490},[177],[7492,7521],{"type":150,"tag":169,"props":7493,"children":7495},{"className":7494},[182],[7496],{"type":150,"tag":185,"props":7497,"children":7498},{"xmlns":187},[7499],{"type":150,"tag":191,"props":7500,"children":7501},{},[7502,7516],{"type":150,"tag":195,"props":7503,"children":7504},{},[7505],{"type":150,"tag":746,"props":7506,"children":7507},{},[7508,7512],{"type":150,"tag":405,"props":7509,"children":7510},{},[7511],{"type":155,"value":5591},{"type":150,"tag":411,"props":7513,"children":7514},{},[7515],{"type":155,"value":433},{"type":150,"tag":223,"props":7517,"children":7518},{"encoding":225},[7519],{"type":155,"value":7520},"N^2",{"type":150,"tag":169,"props":7522,"children":7524},{"className":7523,"ariaHidden":214},[232],[7525],{"type":150,"tag":169,"props":7526,"children":7528},{"className":7527},[237],[7529,7533],{"type":150,"tag":169,"props":7530,"children":7532},{"className":7531,"style":7005},[242],[],{"type":150,"tag":169,"props":7534,"children":7536},{"className":7535},[248],[7537,7542],{"type":150,"tag":169,"props":7538,"children":7540},{"className":7539,"style":5780},[248,489],[7541],{"type":155,"value":5591},{"type":150,"tag":169,"props":7543,"children":7545},{"className":7544},[495],[7546],{"type":150,"tag":169,"props":7547,"children":7549},{"className":7548},[253],[7550],{"type":150,"tag":169,"props":7551,"children":7553},{"className":7552},[259],[7554],{"type":150,"tag":169,"props":7555,"children":7557},{"className":7556,"style":7005},[264],[7558],{"type":150,"tag":169,"props":7559,"children":7560},{"style":888},[7561,7565],{"type":150,"tag":169,"props":7562,"children":7564},{"className":7563,"style":517},[274],[],{"type":150,"tag":169,"props":7566,"children":7568},{"className":7567},[522,523,524,525],[7569],{"type":150,"tag":169,"props":7570,"children":7572},{"className":7571},[248,525],[7573],{"type":155,"value":433},{"type":155,"value":7575}," 压下去，",{"type":150,"tag":169,"props":7577,"children":7579},{"className":7578},[177],[7580,7624],{"type":150,"tag":169,"props":7581,"children":7583},{"className":7582},[182],[7584],{"type":150,"tag":185,"props":7585,"children":7586},{"xmlns":187},[7587],{"type":150,"tag":191,"props":7588,"children":7589},{},[7590,7620],{"type":150,"tag":195,"props":7591,"children":7592},{},[7593,7597,7601,7605,7616],{"type":150,"tag":405,"props":7594,"children":7595},{},[7596],{"type":155,"value":6923},{"type":150,"tag":417,"props":7598,"children":7599},{"stretchy":207},[7600],{"type":155,"value":972},{"type":150,"tag":405,"props":7602,"children":7603},{},[7604],{"type":155,"value":5591},{"type":150,"tag":746,"props":7606,"children":7607},{},[7608,7612],{"type":150,"tag":405,"props":7609,"children":7610},{},[7611],{"type":155,"value":759},{"type":150,"tag":411,"props":7613,"children":7614},{},[7615],{"type":155,"value":433},{"type":150,"tag":417,"props":7617,"children":7618},{"stretchy":207},[7619],{"type":155,"value":982},{"type":150,"tag":223,"props":7621,"children":7622},{"encoding":225},[7623],{"type":155,"value":6951},{"type":150,"tag":169,"props":7625,"children":7627},{"className":7626,"ariaHidden":214},[232],[7628],{"type":150,"tag":169,"props":7629,"children":7631},{"className":7630},[237],[7632,7636,7641,7646,7651,7692],{"type":150,"tag":169,"props":7633,"children":7635},{"className":7634,"style":6963},[242],[],{"type":150,"tag":169,"props":7637,"children":7639},{"className":7638,"style":6968},[248,489],[7640],{"type":155,"value":6923},{"type":150,"tag":169,"props":7642,"children":7644},{"className":7643},[1004],[7645],{"type":155,"value":972},{"type":150,"tag":169,"props":7647,"children":7649},{"className":7648,"style":5780},[248,489],[7650],{"type":155,"value":5591},{"type":150,"tag":169,"props":7652,"children":7654},{"className":7653},[248],[7655,7660],{"type":150,"tag":169,"props":7656,"children":7658},{"className":7657},[248,489],[7659],{"type":155,"value":759},{"type":150,"tag":169,"props":7661,"children":7663},{"className":7662},[495],[7664],{"type":150,"tag":169,"props":7665,"children":7667},{"className":7666},[253],[7668],{"type":150,"tag":169,"props":7669,"children":7671},{"className":7670},[259],[7672],{"type":150,"tag":169,"props":7673,"children":7675},{"className":7674,"style":7005},[264],[7676],{"type":150,"tag":169,"props":7677,"children":7678},{"style":888},[7679,7683],{"type":150,"tag":169,"props":7680,"children":7682},{"className":7681,"style":517},[274],[],{"type":150,"tag":169,"props":7684,"children":7686},{"className":7685},[522,523,524,525],[7687],{"type":150,"tag":169,"props":7688,"children":7690},{"className":7689},[248,525],[7691],{"type":155,"value":433},{"type":150,"tag":169,"props":7693,"children":7695},{"className":7694},[1015],[7696],{"type":155,"value":982},{"type":155,"value":7698}," 这一项就浮到水面上了。第三项 ",{"type":150,"tag":169,"props":7700,"children":7702},{"className":7701},[177],[7703,7732],{"type":150,"tag":169,"props":7704,"children":7706},{"className":7705},[182],[7707],{"type":150,"tag":185,"props":7708,"children":7709},{"xmlns":187},[7710],{"type":150,"tag":191,"props":7711,"children":7712},{},[7713,7727],{"type":150,"tag":195,"props":7714,"children":7715},{},[7716],{"type":150,"tag":401,"props":7717,"children":7718},{},[7719,7723],{"type":150,"tag":405,"props":7720,"children":7721},{},[7722],{"type":155,"value":6294},{"type":150,"tag":217,"props":7724,"children":7725},{},[7726],{"type":155,"value":6347},{"type":150,"tag":223,"props":7728,"children":7729},{"encoding":225},[7730],{"type":155,"value":7731},"C_{\\text{memory movement}}",{"type":150,"tag":169,"props":7733,"children":7735},{"className":7734,"ariaHidden":214},[232],[7736],{"type":150,"tag":169,"props":7737,"children":7739},{"className":7738},[237],[7740,7744],{"type":150,"tag":169,"props":7741,"children":7743},{"className":7742,"style":6629},[242],[],{"type":150,"tag":169,"props":7745,"children":7747},{"className":7746},[248],[7748,7753],{"type":150,"tag":169,"props":7749,"children":7751},{"className":7750,"style":6373},[248,489],[7752],{"type":155,"value":6294},{"type":150,"tag":169,"props":7754,"children":7756},{"className":7755},[495],[7757],{"type":150,"tag":169,"props":7758,"children":7760},{"className":7759},[253,254],[7761,7798],{"type":150,"tag":169,"props":7762,"children":7764},{"className":7763},[259],[7765,7793],{"type":150,"tag":169,"props":7766,"children":7768},{"className":7767,"style":1369},[264],[7769],{"type":150,"tag":169,"props":7770,"children":7771},{"style":6394},[7772,7776],{"type":150,"tag":169,"props":7773,"children":7775},{"className":7774,"style":517},[274],[],{"type":150,"tag":169,"props":7777,"children":7779},{"className":7778},[522,523,524,525],[7780],{"type":150,"tag":169,"props":7781,"children":7783},{"className":7782},[248,525],[7784],{"type":150,"tag":169,"props":7785,"children":7787},{"className":7786},[248,155,525],[7788],{"type":150,"tag":169,"props":7789,"children":7791},{"className":7790},[248,525],[7792],{"type":155,"value":6347},{"type":150,"tag":169,"props":7794,"children":7796},{"className":7795},[355],[7797],{"type":155,"value":358},{"type":150,"tag":169,"props":7799,"children":7801},{"className":7800},[259],[7802],{"type":150,"tag":169,"props":7803,"children":7805},{"className":7804,"style":6692},[264],[7806],{"type":150,"tag":169,"props":7807,"children":7808},{},[],{"type":155,"value":7810}," 是最容易漏的。GPU 上算一个 FLOP 很快，搬一次数据慢得多。",{"type":150,"tag":151,"props":7812,"children":7813},{},[7814],{"type":155,"value":7815},"所以\"减少 KV Cache\"真正的收益，很多时候不是省显存，而是省搬运。但这件事有个限度：decode 每步要读的权重是固定的，搬不动。KV Cache 压到比它更小之后，继续压的边际收益就下降了。",{"type":150,"tag":151,"props":7817,"children":7818},{},[7819],{"type":155,"value":7820},"这条限度解释了后面很多设计的取舍，也解释了为什么 1M context 之后，战场会转向每层的计算和权重。因为账单上还能砍的地方换位置了。",{"type":150,"tag":372,"props":7822,"children":7824},{"id":7823},"mqa-与-gqa",[7825],{"type":155,"value":7826},"MQA 与 GQA",{"type":150,"tag":151,"props":7828,"children":7829},{},[7830,7832,7897,7899,7959,7961,8018],{"type":155,"value":7831},"既然 KV Cache 是问题，最朴素的想法就是：少存点。MQA（Multi-Query Attention）直接让所有 attention head 共享同一份 K、V。用公式说，就是把所有 ",{"type":150,"tag":169,"props":7833,"children":7835},{"className":7834},[177],[7836,7866],{"type":150,"tag":169,"props":7837,"children":7839},{"className":7838},[182],[7840],{"type":150,"tag":185,"props":7841,"children":7842},{"xmlns":187},[7843],{"type":150,"tag":191,"props":7844,"children":7845},{},[7846,7861],{"type":150,"tag":195,"props":7847,"children":7848},{},[7849,7853,7857],{"type":150,"tag":405,"props":7850,"children":7851},{},[7852],{"type":155,"value":1876},{"type":150,"tag":417,"props":7854,"children":7855},{"separator":214},[7856],{"type":155,"value":421},{"type":150,"tag":405,"props":7858,"children":7859},{},[7860],{"type":155,"value":1915},{"type":150,"tag":223,"props":7862,"children":7863},{"encoding":225},[7864],{"type":155,"value":7865},"k, v",{"type":150,"tag":169,"props":7867,"children":7869},{"className":7868,"ariaHidden":214},[232],[7870],{"type":150,"tag":169,"props":7871,"children":7873},{"className":7872},[237],[7874,7878,7883,7888,7892],{"type":150,"tag":169,"props":7875,"children":7877},{"className":7876,"style":1121},[242],[],{"type":150,"tag":169,"props":7879,"children":7881},{"className":7880,"style":2427},[248,489],[7882],{"type":155,"value":1876},{"type":150,"tag":169,"props":7884,"children":7886},{"className":7885},[552],[7887],{"type":155,"value":421},{"type":150,"tag":169,"props":7889,"children":7891},{"className":7890,"style":559},[558],[],{"type":150,"tag":169,"props":7893,"children":7895},{"className":7894,"style":2329},[248,489],[7896],{"type":155,"value":1915},{"type":155,"value":7898}," 的上标 ",{"type":150,"tag":169,"props":7900,"children":7902},{"className":7901},[177],[7903,7932],{"type":150,"tag":169,"props":7904,"children":7906},{"className":7905},[182],[7907],{"type":150,"tag":185,"props":7908,"children":7909},{"xmlns":187},[7910],{"type":150,"tag":191,"props":7911,"children":7912},{},[7913,7928],{"type":150,"tag":195,"props":7914,"children":7915},{},[7916,7920,7924],{"type":150,"tag":417,"props":7917,"children":7918},{"stretchy":207},[7919],{"type":155,"value":972},{"type":150,"tag":405,"props":7921,"children":7922},{},[7923],{"type":155,"value":977},{"type":150,"tag":417,"props":7925,"children":7926},{"stretchy":207},[7927],{"type":155,"value":982},{"type":150,"tag":223,"props":7929,"children":7930},{"encoding":225},[7931],{"type":155,"value":987},{"type":150,"tag":169,"props":7933,"children":7935},{"className":7934,"ariaHidden":214},[232],[7936],{"type":150,"tag":169,"props":7937,"children":7939},{"className":7938},[237],[7940,7944,7949,7954],{"type":150,"tag":169,"props":7941,"children":7943},{"className":7942,"style":999},[242],[],{"type":150,"tag":169,"props":7945,"children":7947},{"className":7946},[1004],[7948],{"type":155,"value":972},{"type":150,"tag":169,"props":7950,"children":7952},{"className":7951},[248,489],[7953],{"type":155,"value":977},{"type":150,"tag":169,"props":7955,"children":7957},{"className":7956},[1015],[7958],{"type":155,"value":982},{"type":155,"value":7960}," 去掉。它是 2019 年提出来的。那会儿 LLM 还没火，可见减少 KV Cache 早就已经是热门课题了。KV Cache 直接降到原来的 ",{"type":150,"tag":169,"props":7962,"children":7964},{"className":7963},[177],[7965,7995],{"type":150,"tag":169,"props":7966,"children":7968},{"className":7967},[182],[7969],{"type":150,"tag":185,"props":7970,"children":7971},{"xmlns":187},[7972],{"type":150,"tag":191,"props":7973,"children":7974},{},[7975,7990],{"type":150,"tag":195,"props":7976,"children":7977},{},[7978,7982,7986],{"type":150,"tag":411,"props":7979,"children":7980},{},[7981],{"type":155,"value":415},{"type":150,"tag":405,"props":7983,"children":7984},{"mathvariant":1796},[7985],{"type":155,"value":4914},{"type":150,"tag":405,"props":7987,"children":7988},{},[7989],{"type":155,"value":926},{"type":150,"tag":223,"props":7991,"children":7992},{"encoding":225},[7993],{"type":155,"value":7994},"1\u002Fh",{"type":150,"tag":169,"props":7996,"children":7998},{"className":7997,"ariaHidden":214},[232],[7999],{"type":150,"tag":169,"props":8000,"children":8002},{"className":8001},[237],[8003,8007,8013],{"type":150,"tag":169,"props":8004,"children":8006},{"className":8005,"style":999},[242],[],{"type":150,"tag":169,"props":8008,"children":8010},{"className":8009},[248],[8011],{"type":155,"value":8012},"1\u002F",{"type":150,"tag":169,"props":8014,"children":8016},{"className":8015},[248,489],[8017],{"type":155,"value":926},{"type":155,"value":8019},"。单从省显存的角度看，这已经是天花板了。",{"type":150,"tag":151,"props":8021,"children":8022},{},[8023],{"type":155,"value":8024},"代价也很直白：原来一个 token 的每个 head 都有各自的 K、V，现在被压成一份，信息损失不小。",{"type":150,"tag":151,"props":8026,"children":8027},{},[8028],{"type":155,"value":8029},"不过 MQA 的支持者认为这个损失被高估了。理由是不同 head 学到的 K、V 本来就高度相关，共享一份丢不掉太多有效信息；而且这部分损失可以通过继续训练补回来。这个论证后来被证明有一定道理，但\"一定道理\"和\"可以无条件依赖\"是两回事。GQA 的出现本身就说明，业界并不敢把压缩率开到 MQA 那么极端。",{"type":150,"tag":151,"props":8031,"children":8032},{},[8033,8035,8077,8079,8122],{"type":155,"value":8034},"有个细节容易被忽略。MQA 共享 K、V 之后，attention 部分的参数量少了将近一半。为了保持总参数量不变，通常会相应增大 FFN 的规模。这能补回来一部分效果损失。但还有人担心压得太狠，影响学习效率和最终效果。于是有了 GQA（Grouped-Query Attention），那是 2023 年的事。GQA 把 ",{"type":150,"tag":169,"props":8036,"children":8038},{"className":8037},[177],[8039,8060],{"type":150,"tag":169,"props":8040,"children":8042},{"className":8041},[182],[8043],{"type":150,"tag":185,"props":8044,"children":8045},{"xmlns":187},[8046],{"type":150,"tag":191,"props":8047,"children":8048},{},[8049,8056],{"type":150,"tag":195,"props":8050,"children":8051},{},[8052],{"type":150,"tag":405,"props":8053,"children":8054},{},[8055],{"type":155,"value":926},{"type":150,"tag":223,"props":8057,"children":8058},{"encoding":225},[8059],{"type":155,"value":926},{"type":150,"tag":169,"props":8061,"children":8063},{"className":8062,"ariaHidden":214},[232],[8064],{"type":150,"tag":169,"props":8065,"children":8067},{"className":8066},[237],[8068,8072],{"type":150,"tag":169,"props":8069,"children":8071},{"className":8070,"style":942},[242],[],{"type":150,"tag":169,"props":8073,"children":8075},{"className":8074},[248,489],[8076],{"type":155,"value":926},{"type":155,"value":8078}," 个 head 分成 ",{"type":150,"tag":169,"props":8080,"children":8082},{"className":8081},[177],[8083,8105],{"type":150,"tag":169,"props":8084,"children":8086},{"className":8085},[182],[8087],{"type":150,"tag":185,"props":8088,"children":8089},{"xmlns":187},[8090],{"type":150,"tag":191,"props":8091,"children":8092},{},[8093,8101],{"type":150,"tag":195,"props":8094,"children":8095},{},[8096],{"type":150,"tag":405,"props":8097,"children":8098},{},[8099],{"type":155,"value":8100},"g",{"type":150,"tag":223,"props":8102,"children":8103},{"encoding":225},[8104],{"type":155,"value":8100},{"type":150,"tag":169,"props":8106,"children":8108},{"className":8107,"ariaHidden":214},[232],[8109],{"type":150,"tag":169,"props":8110,"children":8112},{"className":8111},[237],[8113,8117],{"type":150,"tag":169,"props":8114,"children":8116},{"className":8115,"style":480},[242],[],{"type":150,"tag":169,"props":8118,"children":8120},{"className":8119,"style":2329},[248,489],[8121],{"type":155,"value":8100},{"type":155,"value":8123}," 组，每组共享同一对 K、V。具体怎么分？",{"type":150,"tag":151,"props":8125,"children":8126},{},[8127,8129,8171,8173,8245],{"type":155,"value":8128},"记第 ",{"type":150,"tag":169,"props":8130,"children":8132},{"className":8131},[177],[8133,8154],{"type":150,"tag":169,"props":8134,"children":8136},{"className":8135},[182],[8137],{"type":150,"tag":185,"props":8138,"children":8139},{"xmlns":187},[8140],{"type":150,"tag":191,"props":8141,"children":8142},{},[8143,8150],{"type":150,"tag":195,"props":8144,"children":8145},{},[8146],{"type":150,"tag":405,"props":8147,"children":8148},{},[8149],{"type":155,"value":977},{"type":150,"tag":223,"props":8151,"children":8152},{"encoding":225},[8153],{"type":155,"value":977},{"type":150,"tag":169,"props":8155,"children":8157},{"className":8156,"ariaHidden":214},[232],[8158],{"type":150,"tag":169,"props":8159,"children":8161},{"className":8160},[237],[8162,8166],{"type":150,"tag":169,"props":8163,"children":8165},{"className":8164,"style":1094},[242],[],{"type":150,"tag":169,"props":8167,"children":8169},{"className":8168},[248,489],[8170],{"type":155,"value":977},{"type":155,"value":8172}," 个 head 属于第 ",{"type":150,"tag":169,"props":8174,"children":8176},{"className":8175},[177],[8177,8212],{"type":150,"tag":169,"props":8178,"children":8180},{"className":8179},[182],[8181],{"type":150,"tag":185,"props":8182,"children":8183},{"xmlns":187},[8184],{"type":150,"tag":191,"props":8185,"children":8186},{},[8187,8207],{"type":150,"tag":195,"props":8188,"children":8189},{},[8190,8195,8199,8203],{"type":150,"tag":405,"props":8191,"children":8192},{},[8193],{"type":155,"value":8194},"γ",{"type":150,"tag":417,"props":8196,"children":8197},{"stretchy":207},[8198],{"type":155,"value":972},{"type":150,"tag":405,"props":8200,"children":8201},{},[8202],{"type":155,"value":977},{"type":150,"tag":417,"props":8204,"children":8205},{"stretchy":207},[8206],{"type":155,"value":982},{"type":150,"tag":223,"props":8208,"children":8209},{"encoding":225},[8210],{"type":155,"value":8211},"\\gamma(s)",{"type":150,"tag":169,"props":8213,"children":8215},{"className":8214,"ariaHidden":214},[232],[8216],{"type":150,"tag":169,"props":8217,"children":8219},{"className":8218},[237],[8220,8224,8230,8235,8240],{"type":150,"tag":169,"props":8221,"children":8223},{"className":8222,"style":999},[242],[],{"type":150,"tag":169,"props":8225,"children":8228},{"className":8226,"style":8227},[248,489],"margin-right:0.0556em;",[8229],{"type":155,"value":8194},{"type":150,"tag":169,"props":8231,"children":8233},{"className":8232},[1004],[8234],{"type":155,"value":972},{"type":150,"tag":169,"props":8236,"children":8238},{"className":8237},[248,489],[8239],{"type":155,"value":977},{"type":150,"tag":169,"props":8241,"children":8243},{"className":8242},[1015],[8244],{"type":155,"value":982},{"type":155,"value":8246}," 组：",{"type":150,"tag":169,"props":8248,"children":8250},{"className":8249},[172],[8251],{"type":150,"tag":169,"props":8252,"children":8254},{"className":8253},[177],[8255,8324],{"type":150,"tag":169,"props":8256,"children":8258},{"className":8257},[182],[8259],{"type":150,"tag":185,"props":8260,"children":8261},{"xmlns":187,"display":188},[8262],{"type":150,"tag":191,"props":8263,"children":8264},{},[8265,8319],{"type":150,"tag":195,"props":8266,"children":8267},{},[8268,8272,8276,8280,8284,8288],{"type":150,"tag":405,"props":8269,"children":8270},{},[8271],{"type":155,"value":8194},{"type":150,"tag":417,"props":8273,"children":8274},{"stretchy":207},[8275],{"type":155,"value":972},{"type":150,"tag":405,"props":8277,"children":8278},{},[8279],{"type":155,"value":977},{"type":150,"tag":417,"props":8281,"children":8282},{"stretchy":207},[8283],{"type":155,"value":982},{"type":150,"tag":417,"props":8285,"children":8286},{},[8287],{"type":155,"value":1045},{"type":150,"tag":195,"props":8289,"children":8290},{},[8291,8296,8314],{"type":150,"tag":417,"props":8292,"children":8293},{"fence":214},[8294],{"type":155,"value":8295},"⌈",{"type":150,"tag":1952,"props":8297,"children":8298},{},[8299,8310],{"type":150,"tag":195,"props":8300,"children":8301},{},[8302,8306],{"type":150,"tag":405,"props":8303,"children":8304},{},[8305],{"type":155,"value":977},{"type":150,"tag":405,"props":8307,"children":8308},{},[8309],{"type":155,"value":8100},{"type":150,"tag":405,"props":8311,"children":8312},{},[8313],{"type":155,"value":926},{"type":150,"tag":417,"props":8315,"children":8316},{"fence":214},[8317],{"type":155,"value":8318},"⌉",{"type":150,"tag":223,"props":8320,"children":8321},{"encoding":225},[8322],{"type":155,"value":8323},"\\gamma(s) = \\left\\lceil \\frac{sg}{h} \\right\\rceil",{"type":150,"tag":169,"props":8325,"children":8327},{"className":8326,"ariaHidden":214},[232],[8328,8369],{"type":150,"tag":169,"props":8329,"children":8331},{"className":8330},[237],[8332,8336,8341,8346,8351,8356,8360,8365],{"type":150,"tag":169,"props":8333,"children":8335},{"className":8334,"style":999},[242],[],{"type":150,"tag":169,"props":8337,"children":8339},{"className":8338,"style":8227},[248,489],[8340],{"type":155,"value":8194},{"type":150,"tag":169,"props":8342,"children":8344},{"className":8343},[1004],[8345],{"type":155,"value":972},{"type":150,"tag":169,"props":8347,"children":8349},{"className":8348},[248,489],[8350],{"type":155,"value":977},{"type":150,"tag":169,"props":8352,"children":8354},{"className":8353},[1015],[8355],{"type":155,"value":982},{"type":150,"tag":169,"props":8357,"children":8359},{"className":8358,"style":839},[558],[],{"type":150,"tag":169,"props":8361,"children":8363},{"className":8362},[844],[8364],{"type":155,"value":1045},{"type":150,"tag":169,"props":8366,"children":8368},{"className":8367,"style":839},[558],[],{"type":150,"tag":169,"props":8370,"children":8372},{"className":8371},[237],[8373,8378],{"type":150,"tag":169,"props":8374,"children":8377},{"className":8375,"style":8376},[242],"height:1.836em;vertical-align:-0.686em;",[],{"type":150,"tag":169,"props":8379,"children":8381},{"className":8380},[630],[8382,8395,8493],{"type":150,"tag":169,"props":8383,"children":8387},{"className":8384,"style":8386},[1004,8385],"delimcenter","top:0em;",[8388],{"type":150,"tag":169,"props":8389,"children":8393},{"className":8390},[8391,8392],"delimsizing","size2",[8394],{"type":155,"value":8295},{"type":150,"tag":169,"props":8396,"children":8398},{"className":8397},[248],[8399,8403,8489],{"type":150,"tag":169,"props":8400,"children":8402},{"className":8401},[1004,2660],[],{"type":150,"tag":169,"props":8404,"children":8406},{"className":8405},[1952],[8407],{"type":150,"tag":169,"props":8408,"children":8410},{"className":8409},[253,254],[8411,8477],{"type":150,"tag":169,"props":8412,"children":8414},{"className":8413},[259],[8415,8472],{"type":150,"tag":169,"props":8416,"children":8419},{"className":8417,"style":8418},[264],"height:1.1076em;",[8420,8438,8450],{"type":150,"tag":169,"props":8421,"children":8423},{"style":8422},"top:-2.314em;",[8424,8429],{"type":150,"tag":169,"props":8425,"children":8428},{"className":8426,"style":8427},[274],"height:3em;",[],{"type":150,"tag":169,"props":8430,"children":8432},{"className":8431},[248],[8433],{"type":150,"tag":169,"props":8434,"children":8436},{"className":8435},[248,489],[8437],{"type":155,"value":926},{"type":150,"tag":169,"props":8439,"children":8441},{"style":8440},"top:-3.23em;",[8442,8446],{"type":150,"tag":169,"props":8443,"children":8445},{"className":8444,"style":8427},[274],[],{"type":150,"tag":169,"props":8447,"children":8449},{"className":8448,"style":2982},[2981],[],{"type":150,"tag":169,"props":8451,"children":8453},{"style":8452},"top:-3.677em;",[8454,8458],{"type":150,"tag":169,"props":8455,"children":8457},{"className":8456,"style":8427},[274],[],{"type":150,"tag":169,"props":8459,"children":8461},{"className":8460},[248],[8462,8467],{"type":150,"tag":169,"props":8463,"children":8465},{"className":8464},[248,489],[8466],{"type":155,"value":977},{"type":150,"tag":169,"props":8468,"children":8470},{"className":8469,"style":2329},[248,489],[8471],{"type":155,"value":8100},{"type":150,"tag":169,"props":8473,"children":8475},{"className":8474},[355],[8476],{"type":155,"value":358},{"type":150,"tag":169,"props":8478,"children":8480},{"className":8479},[259],[8481],{"type":150,"tag":169,"props":8482,"children":8485},{"className":8483,"style":8484},[264],"height:0.686em;",[8486],{"type":150,"tag":169,"props":8487,"children":8488},{},[],{"type":150,"tag":169,"props":8490,"children":8492},{"className":8491},[1015,2660],[],{"type":150,"tag":169,"props":8494,"children":8496},{"className":8495,"style":8386},[1015,8385],[8497],{"type":150,"tag":169,"props":8498,"children":8500},{"className":8499},[8391,8392],[8501],{"type":155,"value":8318},{"type":150,"tag":151,"props":8503,"children":8504},{},[8505,8507,8568,8570,8630],{"type":155,"value":8506},"（就是按顺序平均切开：前 ",{"type":150,"tag":169,"props":8508,"children":8510},{"className":8509},[177],[8511,8541],{"type":150,"tag":169,"props":8512,"children":8514},{"className":8513},[182],[8515],{"type":150,"tag":185,"props":8516,"children":8517},{"xmlns":187},[8518],{"type":150,"tag":191,"props":8519,"children":8520},{},[8521,8536],{"type":150,"tag":195,"props":8522,"children":8523},{},[8524,8528,8532],{"type":150,"tag":405,"props":8525,"children":8526},{},[8527],{"type":155,"value":926},{"type":150,"tag":405,"props":8529,"children":8530},{"mathvariant":1796},[8531],{"type":155,"value":4914},{"type":150,"tag":405,"props":8533,"children":8534},{},[8535],{"type":155,"value":8100},{"type":150,"tag":223,"props":8537,"children":8538},{"encoding":225},[8539],{"type":155,"value":8540},"h\u002Fg",{"type":150,"tag":169,"props":8542,"children":8544},{"className":8543,"ariaHidden":214},[232],[8545],{"type":150,"tag":169,"props":8546,"children":8548},{"className":8547},[237],[8549,8553,8558,8563],{"type":150,"tag":169,"props":8550,"children":8552},{"className":8551,"style":999},[242],[],{"type":150,"tag":169,"props":8554,"children":8556},{"className":8555},[248,489],[8557],{"type":155,"value":926},{"type":150,"tag":169,"props":8559,"children":8561},{"className":8560},[248],[8562],{"type":155,"value":4914},{"type":150,"tag":169,"props":8564,"children":8566},{"className":8565,"style":2329},[248,489],[8567],{"type":155,"value":8100},{"type":155,"value":8569}," 个 head 一组，接着 ",{"type":150,"tag":169,"props":8571,"children":8573},{"className":8572},[177],[8574,8603],{"type":150,"tag":169,"props":8575,"children":8577},{"className":8576},[182],[8578],{"type":150,"tag":185,"props":8579,"children":8580},{"xmlns":187},[8581],{"type":150,"tag":191,"props":8582,"children":8583},{},[8584,8599],{"type":150,"tag":195,"props":8585,"children":8586},{},[8587,8591,8595],{"type":150,"tag":405,"props":8588,"children":8589},{},[8590],{"type":155,"value":926},{"type":150,"tag":405,"props":8592,"children":8593},{"mathvariant":1796},[8594],{"type":155,"value":4914},{"type":150,"tag":405,"props":8596,"children":8597},{},[8598],{"type":155,"value":8100},{"type":150,"tag":223,"props":8600,"children":8601},{"encoding":225},[8602],{"type":155,"value":8540},{"type":150,"tag":169,"props":8604,"children":8606},{"className":8605,"ariaHidden":214},[232],[8607],{"type":150,"tag":169,"props":8608,"children":8610},{"className":8609},[237],[8611,8615,8620,8625],{"type":150,"tag":169,"props":8612,"children":8614},{"className":8613,"style":999},[242],[],{"type":150,"tag":169,"props":8616,"children":8618},{"className":8617},[248,489],[8619],{"type":155,"value":926},{"type":150,"tag":169,"props":8621,"children":8623},{"className":8622},[248],[8624],{"type":155,"value":4914},{"type":150,"tag":169,"props":8626,"children":8628},{"className":8627,"style":2329},[248,489],[8629],{"type":155,"value":8100},{"type":155,"value":8631}," 个一组，以此类推。）于是 attention 写成：",{"type":150,"tag":169,"props":8633,"children":8635},{"className":8634},[172],[8636],{"type":150,"tag":169,"props":8637,"children":8639},{"className":8638},[177],[8640,8871],{"type":150,"tag":169,"props":8641,"children":8643},{"className":8642},[182],[8644],{"type":150,"tag":185,"props":8645,"children":8646},{"xmlns":187,"display":188},[8647],{"type":150,"tag":191,"props":8648,"children":8649},{},[8650,8866],{"type":150,"tag":195,"props":8651,"children":8652},{},[8653,8679,8683,8722],{"type":150,"tag":1224,"props":8654,"children":8655},{},[8656,8660,8664],{"type":150,"tag":405,"props":8657,"children":8658},{},[8659],{"type":155,"value":1208},{"type":150,"tag":405,"props":8661,"children":8662},{},[8663],{"type":155,"value":1213},{"type":150,"tag":195,"props":8665,"children":8666},{},[8667,8671,8675],{"type":150,"tag":417,"props":8668,"children":8669},{"stretchy":207},[8670],{"type":155,"value":972},{"type":150,"tag":405,"props":8672,"children":8673},{},[8674],{"type":155,"value":977},{"type":150,"tag":417,"props":8676,"children":8677},{"stretchy":207},[8678],{"type":155,"value":982},{"type":150,"tag":417,"props":8680,"children":8681},{},[8682],{"type":155,"value":1045},{"type":150,"tag":195,"props":8684,"children":8685},{},[8686,8690,8694,8698,8702,8706,8710,8714,8718],{"type":150,"tag":405,"props":8687,"children":8688},{"mathvariant":1796},[8689],{"type":155,"value":1799},{"type":150,"tag":405,"props":8691,"children":8692},{"mathvariant":1796},[8693],{"type":155,"value":1213},{"type":150,"tag":405,"props":8695,"children":8696},{"mathvariant":1796},[8697],{"type":155,"value":1213},{"type":150,"tag":405,"props":8699,"children":8700},{"mathvariant":1796},[8701],{"type":155,"value":1812},{"type":150,"tag":405,"props":8703,"children":8704},{"mathvariant":1796},[8705],{"type":155,"value":1817},{"type":150,"tag":405,"props":8707,"children":8708},{"mathvariant":1796},[8709],{"type":155,"value":1213},{"type":150,"tag":405,"props":8711,"children":8712},{"mathvariant":1796},[8713],{"type":155,"value":739},{"type":150,"tag":405,"props":8715,"children":8716},{"mathvariant":1796},[8717],{"type":155,"value":1208},{"type":150,"tag":405,"props":8719,"children":8720},{"mathvariant":1796},[8721],{"type":155,"value":1817},{"type":150,"tag":195,"props":8723,"children":8724},{},[8725,8729,8755,8759,8764,8809,8813,8817,8862],{"type":150,"tag":417,"props":8726,"children":8727},{"fence":214},[8728],{"type":155,"value":972},{"type":150,"tag":1224,"props":8730,"children":8731},{},[8732,8736,8740],{"type":150,"tag":405,"props":8733,"children":8734},{},[8735],{"type":155,"value":1845},{"type":150,"tag":405,"props":8737,"children":8738},{},[8739],{"type":155,"value":1213},{"type":150,"tag":195,"props":8741,"children":8742},{},[8743,8747,8751],{"type":150,"tag":417,"props":8744,"children":8745},{"stretchy":207},[8746],{"type":155,"value":972},{"type":150,"tag":405,"props":8748,"children":8749},{},[8750],{"type":155,"value":977},{"type":150,"tag":417,"props":8752,"children":8753},{"stretchy":207},[8754],{"type":155,"value":982},{"type":150,"tag":417,"props":8756,"children":8757},{"separator":214},[8758],{"type":155,"value":421},{"type":150,"tag":217,"props":8760,"children":8761},{},[8762],{"type":155,"value":8763}," ",{"type":150,"tag":1224,"props":8765,"children":8766},{},[8767,8771,8782],{"type":150,"tag":405,"props":8768,"children":8769},{},[8770],{"type":155,"value":1876},{"type":150,"tag":195,"props":8772,"children":8773},{},[8774,8778],{"type":150,"tag":417,"props":8775,"children":8776},{},[8777],{"type":155,"value":1884},{"type":150,"tag":405,"props":8779,"children":8780},{},[8781],{"type":155,"value":1213},{"type":150,"tag":195,"props":8783,"children":8784},{},[8785,8789,8793,8797,8801,8805],{"type":150,"tag":417,"props":8786,"children":8787},{"stretchy":207},[8788],{"type":155,"value":972},{"type":150,"tag":405,"props":8790,"children":8791},{},[8792],{"type":155,"value":8194},{"type":150,"tag":417,"props":8794,"children":8795},{"stretchy":207},[8796],{"type":155,"value":972},{"type":150,"tag":405,"props":8798,"children":8799},{},[8800],{"type":155,"value":977},{"type":150,"tag":417,"props":8802,"children":8803},{"stretchy":207},[8804],{"type":155,"value":982},{"type":150,"tag":417,"props":8806,"children":8807},{"stretchy":207},[8808],{"type":155,"value":982},{"type":150,"tag":417,"props":8810,"children":8811},{"separator":214},[8812],{"type":155,"value":421},{"type":150,"tag":217,"props":8814,"children":8815},{},[8816],{"type":155,"value":8763},{"type":150,"tag":1224,"props":8818,"children":8819},{},[8820,8824,8835],{"type":150,"tag":405,"props":8821,"children":8822},{},[8823],{"type":155,"value":1915},{"type":150,"tag":195,"props":8825,"children":8826},{},[8827,8831],{"type":150,"tag":417,"props":8828,"children":8829},{},[8830],{"type":155,"value":1884},{"type":150,"tag":405,"props":8832,"children":8833},{},[8834],{"type":155,"value":1213},{"type":150,"tag":195,"props":8836,"children":8837},{},[8838,8842,8846,8850,8854,8858],{"type":150,"tag":417,"props":8839,"children":8840},{"stretchy":207},[8841],{"type":155,"value":972},{"type":150,"tag":405,"props":8843,"children":8844},{},[8845],{"type":155,"value":8194},{"type":150,"tag":417,"props":8847,"children":8848},{"stretchy":207},[8849],{"type":155,"value":972},{"type":150,"tag":405,"props":8851,"children":8852},{},[8853],{"type":155,"value":977},{"type":150,"tag":417,"props":8855,"children":8856},{"stretchy":207},[8857],{"type":155,"value":982},{"type":150,"tag":417,"props":8859,"children":8860},{"stretchy":207},[8861],{"type":155,"value":982},{"type":150,"tag":417,"props":8863,"children":8864},{"fence":214},[8865],{"type":155,"value":982},{"type":150,"tag":223,"props":8867,"children":8868},{"encoding":225},[8869],{"type":155,"value":8870},"o_t^{(s)} = \\mathrm{Attention}\\left(q_t^{(s)},\\ k^{(\\gamma(s))}_{\\le t},\\ v^{(\\gamma(s))}_{\\le t}\\right)",{"type":150,"tag":169,"props":8872,"children":8874},{"className":8873,"ariaHidden":214},[232],[8875,8983],{"type":150,"tag":169,"props":8876,"children":8878},{"className":8877},[237],[8879,8883,8970,8974,8979],{"type":150,"tag":169,"props":8880,"children":8882},{"className":8881,"style":2196},[242],[],{"type":150,"tag":169,"props":8884,"children":8886},{"className":8885},[248],[8887,8892],{"type":150,"tag":169,"props":8888,"children":8890},{"className":8889},[248,489],[8891],{"type":155,"value":1208},{"type":150,"tag":169,"props":8893,"children":8895},{"className":8894},[495],[8896],{"type":150,"tag":169,"props":8897,"children":8899},{"className":8898},[253,254],[8900,8959],{"type":150,"tag":169,"props":8901,"children":8903},{"className":8902},[259],[8904,8954],{"type":150,"tag":169,"props":8905,"children":8907},{"className":8906,"style":1454},[264],[8908,8924],{"type":150,"tag":169,"props":8909,"children":8910},{"style":1458},[8911,8915],{"type":150,"tag":169,"props":8912,"children":8914},{"className":8913,"style":517},[274],[],{"type":150,"tag":169,"props":8916,"children":8918},{"className":8917},[522,523,524,525],[8919],{"type":150,"tag":169,"props":8920,"children":8922},{"className":8921},[248,489,525],[8923],{"type":155,"value":1213},{"type":150,"tag":169,"props":8925,"children":8926},{"style":1475},[8927,8931],{"type":150,"tag":169,"props":8928,"children":8930},{"className":8929,"style":517},[274],[],{"type":150,"tag":169,"props":8932,"children":8934},{"className":8933},[522,523,524,525],[8935],{"type":150,"tag":169,"props":8936,"children":8938},{"className":8937},[248,525],[8939,8944,8949],{"type":150,"tag":169,"props":8940,"children":8942},{"className":8941},[1004,525],[8943],{"type":155,"value":972},{"type":150,"tag":169,"props":8945,"children":8947},{"className":8946},[248,489,525],[8948],{"type":155,"value":977},{"type":150,"tag":169,"props":8950,"children":8952},{"className":8951},[1015,525],[8953],{"type":155,"value":982},{"type":150,"tag":169,"props":8955,"children":8957},{"className":8956},[355],[8958],{"type":155,"value":358},{"type":150,"tag":169,"props":8960,"children":8962},{"className":8961},[259],[8963],{"type":150,"tag":169,"props":8964,"children":8966},{"className":8965,"style":1516},[264],[8967],{"type":150,"tag":169,"props":8968,"children":8969},{},[],{"type":150,"tag":169,"props":8971,"children":8973},{"className":8972,"style":839},[558],[],{"type":150,"tag":169,"props":8975,"children":8977},{"className":8976},[844],[8978],{"type":155,"value":1045},{"type":150,"tag":169,"props":8980,"children":8982},{"className":8981,"style":839},[558],[],{"type":150,"tag":169,"props":8984,"children":8986},{"className":8985},[237],[8987,8992,9001,9005],{"type":150,"tag":169,"props":8988,"children":8991},{"className":8989,"style":8990},[242],"height:1.8em;vertical-align:-0.65em;",[],{"type":150,"tag":169,"props":8993,"children":8995},{"className":8994},[248],[8996],{"type":150,"tag":169,"props":8997,"children":8999},{"className":8998},[248,2314],[9000],{"type":155,"value":12},{"type":150,"tag":169,"props":9002,"children":9004},{"className":9003,"style":559},[558],[],{"type":150,"tag":169,"props":9006,"children":9008},{"className":9007},[630],[9009,9018,9105,9110,9115,9119,9226,9231,9236,9240,9346],{"type":150,"tag":169,"props":9010,"children":9012},{"className":9011,"style":8386},[1004,8385],[9013],{"type":150,"tag":169,"props":9014,"children":9016},{"className":9015},[8391,8392],[9017],{"type":155,"value":972},{"type":150,"tag":169,"props":9019,"children":9021},{"className":9020},[248],[9022,9027],{"type":150,"tag":169,"props":9023,"children":9025},{"className":9024,"style":2329},[248,489],[9026],{"type":155,"value":1845},{"type":150,"tag":169,"props":9028,"children":9030},{"className":9029},[495],[9031],{"type":150,"tag":169,"props":9032,"children":9034},{"className":9033},[253,254],[9035,9094],{"type":150,"tag":169,"props":9036,"children":9038},{"className":9037},[259],[9039,9089],{"type":150,"tag":169,"props":9040,"children":9042},{"className":9041,"style":1454},[264],[9043,9059],{"type":150,"tag":169,"props":9044,"children":9045},{"style":2350},[9046,9050],{"type":150,"tag":169,"props":9047,"children":9049},{"className":9048,"style":517},[274],[],{"type":150,"tag":169,"props":9051,"children":9053},{"className":9052},[522,523,524,525],[9054],{"type":150,"tag":169,"props":9055,"children":9057},{"className":9056},[248,489,525],[9058],{"type":155,"value":1213},{"type":150,"tag":169,"props":9060,"children":9061},{"style":1475},[9062,9066],{"type":150,"tag":169,"props":9063,"children":9065},{"className":9064,"style":517},[274],[],{"type":150,"tag":169,"props":9067,"children":9069},{"className":9068},[522,523,524,525],[9070],{"type":150,"tag":169,"props":9071,"children":9073},{"className":9072},[248,525],[9074,9079,9084],{"type":150,"tag":169,"props":9075,"children":9077},{"className":9076},[1004,525],[9078],{"type":155,"value":972},{"type":150,"tag":169,"props":9080,"children":9082},{"className":9081},[248,489,525],[9083],{"type":155,"value":977},{"type":150,"tag":169,"props":9085,"children":9087},{"className":9086},[1015,525],[9088],{"type":155,"value":982},{"type":150,"tag":169,"props":9090,"children":9092},{"className":9091},[355],[9093],{"type":155,"value":358},{"type":150,"tag":169,"props":9095,"children":9097},{"className":9096},[259],[9098],{"type":150,"tag":169,"props":9099,"children":9101},{"className":9100,"style":1516},[264],[9102],{"type":150,"tag":169,"props":9103,"children":9104},{},[],{"type":150,"tag":169,"props":9106,"children":9108},{"className":9107},[552],[9109],{"type":155,"value":421},{"type":150,"tag":169,"props":9111,"children":9113},{"className":9112},[558],[9114],{"type":155,"value":8763},{"type":150,"tag":169,"props":9116,"children":9118},{"className":9117,"style":559},[558],[],{"type":150,"tag":169,"props":9120,"children":9122},{"className":9121},[248],[9123,9128],{"type":150,"tag":169,"props":9124,"children":9126},{"className":9125,"style":2427},[248,489],[9127],{"type":155,"value":1876},{"type":150,"tag":169,"props":9129,"children":9131},{"className":9130},[495],[9132],{"type":150,"tag":169,"props":9133,"children":9135},{"className":9134},[253,254],[9136,9215],{"type":150,"tag":169,"props":9137,"children":9139},{"className":9138},[259],[9140,9210],{"type":150,"tag":169,"props":9141,"children":9143},{"className":9142,"style":1454},[264],[9144,9169],{"type":150,"tag":169,"props":9145,"children":9146},{"style":2448},[9147,9151],{"type":150,"tag":169,"props":9148,"children":9150},{"className":9149,"style":517},[274],[],{"type":150,"tag":169,"props":9152,"children":9154},{"className":9153},[522,523,524,525],[9155],{"type":150,"tag":169,"props":9156,"children":9158},{"className":9157},[248,525],[9159,9164],{"type":150,"tag":169,"props":9160,"children":9162},{"className":9161},[844,525],[9163],{"type":155,"value":1884},{"type":150,"tag":169,"props":9165,"children":9167},{"className":9166},[248,489,525],[9168],{"type":155,"value":1213},{"type":150,"tag":169,"props":9170,"children":9171},{"style":1475},[9172,9176],{"type":150,"tag":169,"props":9173,"children":9175},{"className":9174,"style":517},[274],[],{"type":150,"tag":169,"props":9177,"children":9179},{"className":9178},[522,523,524,525],[9180],{"type":150,"tag":169,"props":9181,"children":9183},{"className":9182},[248,525],[9184,9189,9194,9199,9204],{"type":150,"tag":169,"props":9185,"children":9187},{"className":9186},[1004,525],[9188],{"type":155,"value":972},{"type":150,"tag":169,"props":9190,"children":9192},{"className":9191,"style":8227},[248,489,525],[9193],{"type":155,"value":8194},{"type":150,"tag":169,"props":9195,"children":9197},{"className":9196},[1004,525],[9198],{"type":155,"value":972},{"type":150,"tag":169,"props":9200,"children":9202},{"className":9201},[248,489,525],[9203],{"type":155,"value":977},{"type":150,"tag":169,"props":9205,"children":9207},{"className":9206},[1015,525],[9208],{"type":155,"value":9209},"))",{"type":150,"tag":169,"props":9211,"children":9213},{"className":9212},[355],[9214],{"type":155,"value":358},{"type":150,"tag":169,"props":9216,"children":9218},{"className":9217},[259],[9219],{"type":150,"tag":169,"props":9220,"children":9222},{"className":9221,"style":2514},[264],[9223],{"type":150,"tag":169,"props":9224,"children":9225},{},[],{"type":150,"tag":169,"props":9227,"children":9229},{"className":9228},[552],[9230],{"type":155,"value":421},{"type":150,"tag":169,"props":9232,"children":9234},{"className":9233},[558],[9235],{"type":155,"value":8763},{"type":150,"tag":169,"props":9237,"children":9239},{"className":9238,"style":559},[558],[],{"type":150,"tag":169,"props":9241,"children":9243},{"className":9242},[248],[9244,9249],{"type":150,"tag":169,"props":9245,"children":9247},{"className":9246,"style":2329},[248,489],[9248],{"type":155,"value":1915},{"type":150,"tag":169,"props":9250,"children":9252},{"className":9251},[495],[9253],{"type":150,"tag":169,"props":9254,"children":9256},{"className":9255},[253,254],[9257,9335],{"type":150,"tag":169,"props":9258,"children":9260},{"className":9259},[259],[9261,9330],{"type":150,"tag":169,"props":9262,"children":9264},{"className":9263,"style":1454},[264],[9265,9290],{"type":150,"tag":169,"props":9266,"children":9267},{"style":2555},[9268,9272],{"type":150,"tag":169,"props":9269,"children":9271},{"className":9270,"style":517},[274],[],{"type":150,"tag":169,"props":9273,"children":9275},{"className":9274},[522,523,524,525],[9276],{"type":150,"tag":169,"props":9277,"children":9279},{"className":9278},[248,525],[9280,9285],{"type":150,"tag":169,"props":9281,"children":9283},{"className":9282},[844,525],[9284],{"type":155,"value":1884},{"type":150,"tag":169,"props":9286,"children":9288},{"className":9287},[248,489,525],[9289],{"type":155,"value":1213},{"type":150,"tag":169,"props":9291,"children":9292},{"style":1475},[9293,9297],{"type":150,"tag":169,"props":9294,"children":9296},{"className":9295,"style":517},[274],[],{"type":150,"tag":169,"props":9298,"children":9300},{"className":9299},[522,523,524,525],[9301],{"type":150,"tag":169,"props":9302,"children":9304},{"className":9303},[248,525],[9305,9310,9315,9320,9325],{"type":150,"tag":169,"props":9306,"children":9308},{"className":9307},[1004,525],[9309],{"type":155,"value":972},{"type":150,"tag":169,"props":9311,"children":9313},{"className":9312,"style":8227},[248,489,525],[9314],{"type":155,"value":8194},{"type":150,"tag":169,"props":9316,"children":9318},{"className":9317},[1004,525],[9319],{"type":155,"value":972},{"type":150,"tag":169,"props":9321,"children":9323},{"className":9322},[248,489,525],[9324],{"type":155,"value":977},{"type":150,"tag":169,"props":9326,"children":9328},{"className":9327},[1015,525],[9329],{"type":155,"value":9209},{"type":150,"tag":169,"props":9331,"children":9333},{"className":9332},[355],[9334],{"type":155,"value":358},{"type":150,"tag":169,"props":9336,"children":9338},{"className":9337},[259],[9339],{"type":150,"tag":169,"props":9340,"children":9342},{"className":9341,"style":2514},[264],[9343],{"type":150,"tag":169,"props":9344,"children":9345},{},[],{"type":150,"tag":169,"props":9347,"children":9349},{"className":9348,"style":8386},[1015,8385],[9350],{"type":150,"tag":169,"props":9351,"children":9353},{"className":9352},[8391,8392],[9354],{"type":155,"value":982},{"type":150,"tag":151,"props":9356,"children":9357},{},[9358,9360,9438,9440,9517,9519,9596,9598,9675],{"type":155,"value":9359},"注意 Q 仍然是每个 head 一份，只有 K、V 被分组共用。这个不对称就是 GQA 的全部内容。举个具体的数。",{"type":150,"tag":169,"props":9361,"children":9363},{"className":9362},[177],[9364,9395],{"type":150,"tag":169,"props":9365,"children":9367},{"className":9366},[182],[9368],{"type":150,"tag":185,"props":9369,"children":9370},{"xmlns":187},[9371],{"type":150,"tag":191,"props":9372,"children":9373},{},[9374,9390],{"type":150,"tag":195,"props":9375,"children":9376},{},[9377,9381,9385],{"type":150,"tag":405,"props":9378,"children":9379},{},[9380],{"type":155,"value":926},{"type":150,"tag":417,"props":9382,"children":9383},{},[9384],{"type":155,"value":1045},{"type":150,"tag":411,"props":9386,"children":9387},{},[9388],{"type":155,"value":9389},"8",{"type":150,"tag":223,"props":9391,"children":9392},{"encoding":225},[9393],{"type":155,"value":9394},"h = 8",{"type":150,"tag":169,"props":9396,"children":9398},{"className":9397,"ariaHidden":214},[232],[9399,9425],{"type":150,"tag":169,"props":9400,"children":9402},{"className":9401},[237],[9403,9407,9412,9416,9421],{"type":150,"tag":169,"props":9404,"children":9406},{"className":9405,"style":942},[242],[],{"type":150,"tag":169,"props":9408,"children":9410},{"className":9409},[248,489],[9411],{"type":155,"value":926},{"type":150,"tag":169,"props":9413,"children":9415},{"className":9414,"style":839},[558],[],{"type":150,"tag":169,"props":9417,"children":9419},{"className":9418},[844],[9420],{"type":155,"value":1045},{"type":150,"tag":169,"props":9422,"children":9424},{"className":9423,"style":839},[558],[],{"type":150,"tag":169,"props":9426,"children":9428},{"className":9427},[237],[9429,9433],{"type":150,"tag":169,"props":9430,"children":9432},{"className":9431,"style":5186},[242],[],{"type":150,"tag":169,"props":9434,"children":9436},{"className":9435},[248],[9437],{"type":155,"value":9389},{"type":155,"value":9439},"、",{"type":150,"tag":169,"props":9441,"children":9443},{"className":9442},[177],[9444,9474],{"type":150,"tag":169,"props":9445,"children":9447},{"className":9446},[182],[9448],{"type":150,"tag":185,"props":9449,"children":9450},{"xmlns":187},[9451],{"type":150,"tag":191,"props":9452,"children":9453},{},[9454,9469],{"type":150,"tag":195,"props":9455,"children":9456},{},[9457,9461,9465],{"type":150,"tag":405,"props":9458,"children":9459},{},[9460],{"type":155,"value":8100},{"type":150,"tag":417,"props":9462,"children":9463},{},[9464],{"type":155,"value":1045},{"type":150,"tag":411,"props":9466,"children":9467},{},[9468],{"type":155,"value":433},{"type":150,"tag":223,"props":9470,"children":9471},{"encoding":225},[9472],{"type":155,"value":9473},"g = 2",{"type":150,"tag":169,"props":9475,"children":9477},{"className":9476,"ariaHidden":214},[232],[9478,9504],{"type":150,"tag":169,"props":9479,"children":9481},{"className":9480},[237],[9482,9486,9491,9495,9500],{"type":150,"tag":169,"props":9483,"children":9485},{"className":9484,"style":480},[242],[],{"type":150,"tag":169,"props":9487,"children":9489},{"className":9488,"style":2329},[248,489],[9490],{"type":155,"value":8100},{"type":150,"tag":169,"props":9492,"children":9494},{"className":9493,"style":839},[558],[],{"type":150,"tag":169,"props":9496,"children":9498},{"className":9497},[844],[9499],{"type":155,"value":1045},{"type":150,"tag":169,"props":9501,"children":9503},{"className":9502,"style":839},[558],[],{"type":150,"tag":169,"props":9505,"children":9507},{"className":9506},[237],[9508,9512],{"type":150,"tag":169,"props":9509,"children":9511},{"className":9510,"style":5186},[242],[],{"type":150,"tag":169,"props":9513,"children":9515},{"className":9514},[248],[9516],{"type":155,"value":433},{"type":155,"value":9518}," 时，第 1 到第 4 个 head 共用一对 K、V，第 5 到第 8 个 head 共用另一对。",{"type":150,"tag":169,"props":9520,"children":9522},{"className":9521},[177],[9523,9553],{"type":150,"tag":169,"props":9524,"children":9526},{"className":9525},[182],[9527],{"type":150,"tag":185,"props":9528,"children":9529},{"xmlns":187},[9530],{"type":150,"tag":191,"props":9531,"children":9532},{},[9533,9548],{"type":150,"tag":195,"props":9534,"children":9535},{},[9536,9540,9544],{"type":150,"tag":405,"props":9537,"children":9538},{},[9539],{"type":155,"value":8100},{"type":150,"tag":417,"props":9541,"children":9542},{},[9543],{"type":155,"value":1045},{"type":150,"tag":405,"props":9545,"children":9546},{},[9547],{"type":155,"value":926},{"type":150,"tag":223,"props":9549,"children":9550},{"encoding":225},[9551],{"type":155,"value":9552},"g = h",{"type":150,"tag":169,"props":9554,"children":9556},{"className":9555,"ariaHidden":214},[232],[9557,9583],{"type":150,"tag":169,"props":9558,"children":9560},{"className":9559},[237],[9561,9565,9570,9574,9579],{"type":150,"tag":169,"props":9562,"children":9564},{"className":9563,"style":480},[242],[],{"type":150,"tag":169,"props":9566,"children":9568},{"className":9567,"style":2329},[248,489],[9569],{"type":155,"value":8100},{"type":150,"tag":169,"props":9571,"children":9573},{"className":9572,"style":839},[558],[],{"type":150,"tag":169,"props":9575,"children":9577},{"className":9576},[844],[9578],{"type":155,"value":1045},{"type":150,"tag":169,"props":9580,"children":9582},{"className":9581,"style":839},[558],[],{"type":150,"tag":169,"props":9584,"children":9586},{"className":9585},[237],[9587,9591],{"type":150,"tag":169,"props":9588,"children":9590},{"className":9589,"style":942},[242],[],{"type":150,"tag":169,"props":9592,"children":9594},{"className":9593},[248,489],[9595],{"type":155,"value":926},{"type":155,"value":9597}," 时每个 head 各自一对，退化成 MHA；",{"type":150,"tag":169,"props":9599,"children":9601},{"className":9600},[177],[9602,9632],{"type":150,"tag":169,"props":9603,"children":9605},{"className":9604},[182],[9606],{"type":150,"tag":185,"props":9607,"children":9608},{"xmlns":187},[9609],{"type":150,"tag":191,"props":9610,"children":9611},{},[9612,9627],{"type":150,"tag":195,"props":9613,"children":9614},{},[9615,9619,9623],{"type":150,"tag":405,"props":9616,"children":9617},{},[9618],{"type":155,"value":8100},{"type":150,"tag":417,"props":9620,"children":9621},{},[9622],{"type":155,"value":1045},{"type":150,"tag":411,"props":9624,"children":9625},{},[9626],{"type":155,"value":415},{"type":150,"tag":223,"props":9628,"children":9629},{"encoding":225},[9630],{"type":155,"value":9631},"g = 1",{"type":150,"tag":169,"props":9633,"children":9635},{"className":9634,"ariaHidden":214},[232],[9636,9662],{"type":150,"tag":169,"props":9637,"children":9639},{"className":9638},[237],[9640,9644,9649,9653,9658],{"type":150,"tag":169,"props":9641,"children":9643},{"className":9642,"style":480},[242],[],{"type":150,"tag":169,"props":9645,"children":9647},{"className":9646,"style":2329},[248,489],[9648],{"type":155,"value":8100},{"type":150,"tag":169,"props":9650,"children":9652},{"className":9651,"style":839},[558],[],{"type":150,"tag":169,"props":9654,"children":9656},{"className":9655},[844],[9657],{"type":155,"value":1045},{"type":150,"tag":169,"props":9659,"children":9661},{"className":9660,"style":839},[558],[],{"type":150,"tag":169,"props":9663,"children":9665},{"className":9664},[237],[9666,9670],{"type":150,"tag":169,"props":9667,"children":9669},{"className":9668,"style":5186},[242],[],{"type":150,"tag":169,"props":9671,"children":9673},{"className":9672},[248],[9674],{"type":155,"value":415},{"type":155,"value":9676}," 时所有 head 共用一对，退化成 MQA。",{"type":150,"tag":151,"props":9678,"children":9679},{},[9680],{"type":155,"value":9681},"它提供了 MHA 到 MQA 的自然过渡，压缩率不如 MQA，但自由度和效果保证更好。",{"type":150,"tag":151,"props":9683,"children":9684},{},[9685,9687,9764,9766,9842],{"type":155,"value":9686},"llama2\u002F3-70B 用的 ",{"type":150,"tag":169,"props":9688,"children":9690},{"className":9689},[177],[9691,9721],{"type":150,"tag":169,"props":9692,"children":9694},{"className":9693},[182],[9695],{"type":150,"tag":185,"props":9696,"children":9697},{"xmlns":187},[9698],{"type":150,"tag":191,"props":9699,"children":9700},{},[9701,9716],{"type":150,"tag":195,"props":9702,"children":9703},{},[9704,9708,9712],{"type":150,"tag":405,"props":9705,"children":9706},{},[9707],{"type":155,"value":8100},{"type":150,"tag":417,"props":9709,"children":9710},{},[9711],{"type":155,"value":1045},{"type":150,"tag":411,"props":9713,"children":9714},{},[9715],{"type":155,"value":9389},{"type":150,"tag":223,"props":9717,"children":9718},{"encoding":225},[9719],{"type":155,"value":9720},"g = 8",{"type":150,"tag":169,"props":9722,"children":9724},{"className":9723,"ariaHidden":214},[232],[9725,9751],{"type":150,"tag":169,"props":9726,"children":9728},{"className":9727},[237],[9729,9733,9738,9742,9747],{"type":150,"tag":169,"props":9730,"children":9732},{"className":9731,"style":480},[242],[],{"type":150,"tag":169,"props":9734,"children":9736},{"className":9735,"style":2329},[248,489],[9737],{"type":155,"value":8100},{"type":150,"tag":169,"props":9739,"children":9741},{"className":9740,"style":839},[558],[],{"type":150,"tag":169,"props":9743,"children":9745},{"className":9744},[844],[9746],{"type":155,"value":1045},{"type":150,"tag":169,"props":9748,"children":9750},{"className":9749,"style":839},[558],[],{"type":150,"tag":169,"props":9752,"children":9754},{"className":9753},[237],[9755,9759],{"type":150,"tag":169,"props":9756,"children":9758},{"className":9757,"style":5186},[242],[],{"type":150,"tag":169,"props":9760,"children":9762},{"className":9761},[248],[9763],{"type":155,"value":9389},{"type":155,"value":9765},"。早年有个很整齐的解释：70B 这个体量放不进单卡，得单机 8 卡，而 attention 的 head 本来就是独立运算再拼接的，",{"type":150,"tag":169,"props":9767,"children":9769},{"className":9768},[177],[9770,9799],{"type":150,"tag":169,"props":9771,"children":9773},{"className":9772},[182],[9774],{"type":150,"tag":185,"props":9775,"children":9776},{"xmlns":187},[9777],{"type":150,"tag":191,"props":9778,"children":9779},{},[9780,9795],{"type":150,"tag":195,"props":9781,"children":9782},{},[9783,9787,9791],{"type":150,"tag":405,"props":9784,"children":9785},{},[9786],{"type":155,"value":8100},{"type":150,"tag":417,"props":9788,"children":9789},{},[9790],{"type":155,"value":1045},{"type":150,"tag":411,"props":9792,"children":9793},{},[9794],{"type":155,"value":9389},{"type":150,"tag":223,"props":9796,"children":9797},{"encoding":225},[9798],{"type":155,"value":9720},{"type":150,"tag":169,"props":9800,"children":9802},{"className":9801,"ariaHidden":214},[232],[9803,9829],{"type":150,"tag":169,"props":9804,"children":9806},{"className":9805},[237],[9807,9811,9816,9820,9825],{"type":150,"tag":169,"props":9808,"children":9810},{"className":9809,"style":480},[242],[],{"type":150,"tag":169,"props":9812,"children":9814},{"className":9813,"style":2329},[248,489],[9815],{"type":155,"value":8100},{"type":150,"tag":169,"props":9817,"children":9819},{"className":9818,"style":839},[558],[],{"type":150,"tag":169,"props":9821,"children":9823},{"className":9822},[844],[9824],{"type":155,"value":1045},{"type":150,"tag":169,"props":9826,"children":9828},{"className":9827,"style":839},[558],[],{"type":150,"tag":169,"props":9830,"children":9832},{"className":9831},[237],[9833,9837],{"type":150,"tag":169,"props":9834,"children":9836},{"className":9835,"style":5186},[242],[],{"type":150,"tag":169,"props":9838,"children":9840},{"className":9839},[248],[9841],{"type":155,"value":9389},{"type":155,"value":9843}," 正好让每张卡负责一组 K、V 对应的 head，把卡间通信压到最小。这个解释现在不太站得住了。量化让更大的模型能进单卡，MoE 让 activated 参数远小于总参数，跨机推理也早就不是禁区。",{"type":150,"tag":151,"props":9845,"children":9846},{},[9847,9849,9858],{"type":155,"value":9848},"两个现成的例子：Qwen3.8-Flash-Next 是 125B 的模型、每个 token 只激活 6B，在 B200 上用 TP4 就能跑（",{"type":150,"tag":9850,"props":9851,"children":9855},"a",{"href":9852,"rel":9853},"https:\u002F\u002Fwww.lmsys.org\u002Fblog\u002F2026-08-26-qwen-flash-next",[9854],"nofollow",[9856],{"type":155,"value":9857},"SGLang 的 day-0 支持",{"type":155,"value":9859},"）；DeepSeek-V4-Flash-FP8 同样 TP4 就能跑，H200 上就够。",{"type":150,"tag":151,"props":9861,"children":9862},{},[9863,9865,9907,9909,9985],{"type":155,"value":9864},"所以更准确的说法是：",{"type":150,"tag":169,"props":9866,"children":9868},{"className":9867},[177],[9869,9890],{"type":150,"tag":169,"props":9870,"children":9872},{"className":9871},[182],[9873],{"type":150,"tag":185,"props":9874,"children":9875},{"xmlns":187},[9876],{"type":150,"tag":191,"props":9877,"children":9878},{},[9879,9886],{"type":150,"tag":195,"props":9880,"children":9881},{},[9882],{"type":150,"tag":405,"props":9883,"children":9884},{},[9885],{"type":155,"value":8100},{"type":150,"tag":223,"props":9887,"children":9888},{"encoding":225},[9889],{"type":155,"value":8100},{"type":150,"tag":169,"props":9891,"children":9893},{"className":9892,"ariaHidden":214},[232],[9894],{"type":150,"tag":169,"props":9895,"children":9897},{"className":9896},[237],[9898,9902],{"type":150,"tag":169,"props":9899,"children":9901},{"className":9900,"style":480},[242],[],{"type":150,"tag":169,"props":9903,"children":9905},{"className":9904,"style":2329},[248,489],[9906],{"type":155,"value":8100},{"type":155,"value":9908}," 跟并行度对齐仍然是个好主意，但\"必须单机 8 卡\"这种硬约束已经没有了。",{"type":150,"tag":169,"props":9910,"children":9912},{"className":9911},[177],[9913,9942],{"type":150,"tag":169,"props":9914,"children":9916},{"className":9915},[182],[9917],{"type":150,"tag":185,"props":9918,"children":9919},{"xmlns":187},[9920],{"type":150,"tag":191,"props":9921,"children":9922},{},[9923,9938],{"type":150,"tag":195,"props":9924,"children":9925},{},[9926,9930,9934],{"type":150,"tag":405,"props":9927,"children":9928},{},[9929],{"type":155,"value":8100},{"type":150,"tag":417,"props":9931,"children":9932},{},[9933],{"type":155,"value":1045},{"type":150,"tag":411,"props":9935,"children":9936},{},[9937],{"type":155,"value":9389},{"type":150,"tag":223,"props":9939,"children":9940},{"encoding":225},[9941],{"type":155,"value":9720},{"type":150,"tag":169,"props":9943,"children":9945},{"className":9944,"ariaHidden":214},[232],[9946,9972],{"type":150,"tag":169,"props":9947,"children":9949},{"className":9948},[237],[9950,9954,9959,9963,9968],{"type":150,"tag":169,"props":9951,"children":9953},{"className":9952,"style":480},[242],[],{"type":150,"tag":169,"props":9955,"children":9957},{"className":9956,"style":2329},[248,489],[9958],{"type":155,"value":8100},{"type":150,"tag":169,"props":9960,"children":9962},{"className":9961,"style":839},[558],[],{"type":150,"tag":169,"props":9964,"children":9966},{"className":9965},[844],[9967],{"type":155,"value":1045},{"type":150,"tag":169,"props":9969,"children":9971},{"className":9970,"style":839},[558],[],{"type":150,"tag":169,"props":9973,"children":9975},{"className":9974},[237],[9976,9980],{"type":150,"tag":169,"props":9977,"children":9979},{"className":9978,"style":5186},[242],[],{"type":150,"tag":169,"props":9981,"children":9983},{"className":9982},[248],[9984],{"type":155,"value":9389},{"type":155,"value":9986}," 能留下来，一半是因为并行度，一半是习惯。但 MQA 和 GQA 有一个共同的性质，值得单独点出来：",{"type":150,"tag":151,"props":9988,"children":9989},{},[9990],{"type":150,"tag":158,"props":9991,"children":9992},{},[9993],{"type":155,"value":9994},"它们压的是 head 的数量，不是 head 的内容。",{"type":150,"tag":151,"props":9996,"children":9997},{},[9998,10000,10005],{"type":155,"value":9999},"被保留下来的那些 KV，每一个仍然是完整的、未经压缩的向量。共享是",{"type":150,"tag":158,"props":10001,"children":10002},{},[10003],{"type":155,"value":10004},"离散",{"type":155,"value":10006},"的。要么整份共用，要么各自一份，中间没有别的可能。下一节就是打破这个限制。",{"type":150,"tag":372,"props":10008,"children":10010},{"id":10009},"mla",[10011],{"type":155,"value":13},{"type":150,"tag":151,"props":10013,"children":10014},{},[10015],{"type":155,"value":10016},"MLA（Multi-head Latent Attention）是 DeepSeek-V2 在 2024 年 5 月提出来的，到现在两年多。技术报告是从低秩投影的角度引入的，以至于有人问：LoRA 提出这么久了，为什么直到 MLA 才有人对 KV Cache 做低秩分解？我认为低秩投影这个角度不贴近本质。因为要说低秩投影的话，只要把 GQA 的所有 K、V 拼成一个长向量，就会发现 GQA 本身也是低秩投影。",{"type":150,"tag":151,"props":10018,"children":10019},{},[10020,10022,10125,10127,10169],{"type":155,"value":10021},"把这个拼起来的向量记作 ",{"type":150,"tag":169,"props":10023,"children":10025},{"className":10024},[177],[10026,10056],{"type":150,"tag":169,"props":10027,"children":10029},{"className":10028},[182],[10030],{"type":150,"tag":185,"props":10031,"children":10032},{"xmlns":187},[10033],{"type":150,"tag":191,"props":10034,"children":10035},{},[10036,10051],{"type":150,"tag":195,"props":10037,"children":10038},{},[10039],{"type":150,"tag":401,"props":10040,"children":10041},{},[10042,10047],{"type":150,"tag":405,"props":10043,"children":10044},{},[10045],{"type":155,"value":10046},"c",{"type":150,"tag":405,"props":10048,"children":10049},{},[10050],{"type":155,"value":739},{"type":150,"tag":223,"props":10052,"children":10053},{"encoding":225},[10054],{"type":155,"value":10055},"c_i",{"type":150,"tag":169,"props":10057,"children":10059},{"className":10058,"ariaHidden":214},[232],[10060],{"type":150,"tag":169,"props":10061,"children":10063},{"className":10062},[237],[10064,10068],{"type":150,"tag":169,"props":10065,"children":10067},{"className":10066,"style":1343},[242],[],{"type":150,"tag":169,"props":10069,"children":10071},{"className":10070},[248],[10072,10077],{"type":150,"tag":169,"props":10073,"children":10075},{"className":10074},[248,489],[10076],{"type":155,"value":10046},{"type":150,"tag":169,"props":10078,"children":10080},{"className":10079},[495],[10081],{"type":150,"tag":169,"props":10082,"children":10084},{"className":10083},[253,254],[10085,10114],{"type":150,"tag":169,"props":10086,"children":10088},{"className":10087},[259],[10089,10109],{"type":150,"tag":169,"props":10090,"children":10092},{"className":10091,"style":802},[264],[10093],{"type":150,"tag":169,"props":10094,"children":10095},{"style":512},[10096,10100],{"type":150,"tag":169,"props":10097,"children":10099},{"className":10098,"style":517},[274],[],{"type":150,"tag":169,"props":10101,"children":10103},{"className":10102},[522,523,524,525],[10104],{"type":150,"tag":169,"props":10105,"children":10107},{"className":10106},[248,489,525],[10108],{"type":155,"value":739},{"type":150,"tag":169,"props":10110,"children":10112},{"className":10111},[355],[10113],{"type":155,"value":358},{"type":150,"tag":169,"props":10115,"children":10117},{"className":10116},[259],[10118],{"type":150,"tag":169,"props":10119,"children":10121},{"className":10120,"style":544},[264],[10122],{"type":150,"tag":169,"props":10123,"children":10124},{},[],{"type":155,"value":10126},"。它由 ",{"type":150,"tag":169,"props":10128,"children":10130},{"className":10129},[177],[10131,10152],{"type":150,"tag":169,"props":10132,"children":10134},{"className":10133},[182],[10135],{"type":150,"tag":185,"props":10136,"children":10137},{"xmlns":187},[10138],{"type":150,"tag":191,"props":10139,"children":10140},{},[10141,10148],{"type":150,"tag":195,"props":10142,"children":10143},{},[10144],{"type":150,"tag":405,"props":10145,"children":10146},{},[10147],{"type":155,"value":8100},{"type":150,"tag":223,"props":10149,"children":10150},{"encoding":225},[10151],{"type":155,"value":8100},{"type":150,"tag":169,"props":10153,"children":10155},{"className":10154,"ariaHidden":214},[232],[10156],{"type":150,"tag":169,"props":10157,"children":10159},{"className":10158},[237],[10160,10164],{"type":150,"tag":169,"props":10161,"children":10163},{"className":10162,"style":480},[242],[],{"type":150,"tag":169,"props":10165,"children":10167},{"className":10166,"style":2329},[248,489],[10168],{"type":155,"value":8100},{"type":155,"value":10170}," 组 K、V 依次首尾相接而成：",{"type":150,"tag":169,"props":10172,"children":10174},{"className":10173},[172],[10175],{"type":150,"tag":169,"props":10176,"children":10178},{"className":10177},[177],[10179,10412],{"type":150,"tag":169,"props":10180,"children":10182},{"className":10181},[182],[10183],{"type":150,"tag":185,"props":10184,"children":10185},{"xmlns":187,"display":188},[10186],{"type":150,"tag":191,"props":10187,"children":10188},{},[10189,10407],{"type":150,"tag":195,"props":10190,"children":10191},{},[10192,10203,10207,10211,10237,10241,10245,10249,10253,10279,10283,10309,10313,10317,10321,10325,10351,10355,10359],{"type":150,"tag":401,"props":10193,"children":10194},{},[10195,10199],{"type":150,"tag":405,"props":10196,"children":10197},{},[10198],{"type":155,"value":10046},{"type":150,"tag":405,"props":10200,"children":10201},{},[10202],{"type":155,"value":739},{"type":150,"tag":417,"props":10204,"children":10205},{},[10206],{"type":155,"value":1045},{"type":150,"tag":417,"props":10208,"children":10209},{"stretchy":207},[10210],{"type":155,"value":1222},{"type":150,"tag":1224,"props":10212,"children":10213},{},[10214,10218,10222],{"type":150,"tag":405,"props":10215,"children":10216},{},[10217],{"type":155,"value":1876},{"type":150,"tag":405,"props":10219,"children":10220},{},[10221],{"type":155,"value":739},{"type":150,"tag":195,"props":10223,"children":10224},{},[10225,10229,10233],{"type":150,"tag":417,"props":10226,"children":10227},{"stretchy":207},[10228],{"type":155,"value":972},{"type":150,"tag":411,"props":10230,"children":10231},{},[10232],{"type":155,"value":415},{"type":150,"tag":417,"props":10234,"children":10235},{"stretchy":207},[10236],{"type":155,"value":982},{"type":150,"tag":417,"props":10238,"children":10239},{"separator":214},[10240],{"type":155,"value":421},{"type":150,"tag":417,"props":10242,"children":10243},{},[10244],{"type":155,"value":442},{"type":150,"tag":217,"props":10246,"children":10247},{},[10248],{"type":155,"value":447},{"type":150,"tag":417,"props":10250,"children":10251},{"separator":214},[10252],{"type":155,"value":421},{"type":150,"tag":1224,"props":10254,"children":10255},{},[10256,10260,10264],{"type":150,"tag":405,"props":10257,"children":10258},{},[10259],{"type":155,"value":1876},{"type":150,"tag":405,"props":10261,"children":10262},{},[10263],{"type":155,"value":739},{"type":150,"tag":195,"props":10265,"children":10266},{},[10267,10271,10275],{"type":150,"tag":417,"props":10268,"children":10269},{"stretchy":207},[10270],{"type":155,"value":972},{"type":150,"tag":405,"props":10272,"children":10273},{},[10274],{"type":155,"value":8100},{"type":150,"tag":417,"props":10276,"children":10277},{"stretchy":207},[10278],{"type":155,"value":982},{"type":150,"tag":417,"props":10280,"children":10281},{"separator":214},[10282],{"type":155,"value":421},{"type":150,"tag":1224,"props":10284,"children":10285},{},[10286,10290,10294],{"type":150,"tag":405,"props":10287,"children":10288},{},[10289],{"type":155,"value":1915},{"type":150,"tag":405,"props":10291,"children":10292},{},[10293],{"type":155,"value":739},{"type":150,"tag":195,"props":10295,"children":10296},{},[10297,10301,10305],{"type":150,"tag":417,"props":10298,"children":10299},{"stretchy":207},[10300],{"type":155,"value":972},{"type":150,"tag":411,"props":10302,"children":10303},{},[10304],{"type":155,"value":415},{"type":150,"tag":417,"props":10306,"children":10307},{"stretchy":207},[10308],{"type":155,"value":982},{"type":150,"tag":417,"props":10310,"children":10311},{"separator":214},[10312],{"type":155,"value":421},{"type":150,"tag":417,"props":10314,"children":10315},{},[10316],{"type":155,"value":442},{"type":150,"tag":217,"props":10318,"children":10319},{},[10320],{"type":155,"value":447},{"type":150,"tag":417,"props":10322,"children":10323},{"separator":214},[10324],{"type":155,"value":421},{"type":150,"tag":1224,"props":10326,"children":10327},{},[10328,10332,10336],{"type":150,"tag":405,"props":10329,"children":10330},{},[10331],{"type":155,"value":1915},{"type":150,"tag":405,"props":10333,"children":10334},{},[10335],{"type":155,"value":739},{"type":150,"tag":195,"props":10337,"children":10338},{},[10339,10343,10347],{"type":150,"tag":417,"props":10340,"children":10341},{"stretchy":207},[10342],{"type":155,"value":972},{"type":150,"tag":405,"props":10344,"children":10345},{},[10346],{"type":155,"value":8100},{"type":150,"tag":417,"props":10348,"children":10349},{"stretchy":207},[10350],{"type":155,"value":982},{"type":150,"tag":417,"props":10352,"children":10353},{"stretchy":207},[10354],{"type":155,"value":1326},{"type":150,"tag":417,"props":10356,"children":10357},{},[10358],{"type":155,"value":744},{"type":150,"tag":746,"props":10360,"children":10361},{},[10362,10366],{"type":150,"tag":405,"props":10363,"children":10364},{"mathvariant":751},[10365],{"type":155,"value":754},{"type":150,"tag":195,"props":10367,"children":10368},{},[10369,10373,10377,10388,10392,10403],{"type":150,"tag":405,"props":10370,"children":10371},{},[10372],{"type":155,"value":8100},{"type":150,"tag":417,"props":10374,"children":10375},{"stretchy":207},[10376],{"type":155,"value":972},{"type":150,"tag":401,"props":10378,"children":10379},{},[10380,10384],{"type":150,"tag":405,"props":10381,"children":10382},{},[10383],{"type":155,"value":759},{"type":150,"tag":405,"props":10385,"children":10386},{},[10387],{"type":155,"value":1876},{"type":150,"tag":417,"props":10389,"children":10390},{},[10391],{"type":155,"value":5136},{"type":150,"tag":401,"props":10393,"children":10394},{},[10395,10399],{"type":150,"tag":405,"props":10396,"children":10397},{},[10398],{"type":155,"value":759},{"type":150,"tag":405,"props":10400,"children":10401},{},[10402],{"type":155,"value":1915},{"type":150,"tag":417,"props":10404,"children":10405},{"stretchy":207},[10406],{"type":155,"value":982},{"type":150,"tag":223,"props":10408,"children":10409},{"encoding":225},[10410],{"type":155,"value":10411},"c_i = [k_i^{(1)}, \\cdots, k_i^{(g)}, v_i^{(1)}, \\cdots, v_i^{(g)}] \\in \\mathbb{R}^{g(d_k + d_v)}",{"type":150,"tag":169,"props":10413,"children":10415},{"className":10414,"ariaHidden":214},[232],[10416,10494,10944],{"type":150,"tag":169,"props":10417,"children":10419},{"className":10418},[237],[10420,10424,10481,10485,10490],{"type":150,"tag":169,"props":10421,"children":10423},{"className":10422,"style":1343},[242],[],{"type":150,"tag":169,"props":10425,"children":10427},{"className":10426},[248],[10428,10433],{"type":150,"tag":169,"props":10429,"children":10431},{"className":10430},[248,489],[10432],{"type":155,"value":10046},{"type":150,"tag":169,"props":10434,"children":10436},{"className":10435},[495],[10437],{"type":150,"tag":169,"props":10438,"children":10440},{"className":10439},[253,254],[10441,10470],{"type":150,"tag":169,"props":10442,"children":10444},{"className":10443},[259],[10445,10465],{"type":150,"tag":169,"props":10446,"children":10448},{"className":10447,"style":802},[264],[10449],{"type":150,"tag":169,"props":10450,"children":10451},{"style":512},[10452,10456],{"type":150,"tag":169,"props":10453,"children":10455},{"className":10454,"style":517},[274],[],{"type":150,"tag":169,"props":10457,"children":10459},{"className":10458},[522,523,524,525],[10460],{"type":150,"tag":169,"props":10461,"children":10463},{"className":10462},[248,489,525],[10464],{"type":155,"value":739},{"type":150,"tag":169,"props":10466,"children":10468},{"className":10467},[355],[10469],{"type":155,"value":358},{"type":150,"tag":169,"props":10471,"children":10473},{"className":10472},[259],[10474],{"type":150,"tag":169,"props":10475,"children":10477},{"className":10476,"style":544},[264],[10478],{"type":150,"tag":169,"props":10479,"children":10480},{},[],{"type":150,"tag":169,"props":10482,"children":10484},{"className":10483,"style":839},[558],[],{"type":150,"tag":169,"props":10486,"children":10488},{"className":10487},[844],[10489],{"type":155,"value":1045},{"type":150,"tag":169,"props":10491,"children":10493},{"className":10492,"style":839},[558],[],{"type":150,"tag":169,"props":10495,"children":10497},{"className":10496},[237],[10498,10502,10507,10594,10599,10603,10608,10612,10616,10621,10625,10712,10717,10721,10808,10813,10817,10822,10826,10830,10835,10839,10926,10931,10935,10940],{"type":150,"tag":169,"props":10499,"children":10501},{"className":10500,"style":3694},[242],[],{"type":150,"tag":169,"props":10503,"children":10505},{"className":10504},[1004],[10506],{"type":155,"value":1222},{"type":150,"tag":169,"props":10508,"children":10510},{"className":10509},[248],[10511,10516],{"type":150,"tag":169,"props":10512,"children":10514},{"className":10513,"style":2427},[248,489],[10515],{"type":155,"value":1876},{"type":150,"tag":169,"props":10517,"children":10519},{"className":10518},[495],[10520],{"type":150,"tag":169,"props":10521,"children":10523},{"className":10522},[253,254],[10524,10583],{"type":150,"tag":169,"props":10525,"children":10527},{"className":10526},[259],[10528,10578],{"type":150,"tag":169,"props":10529,"children":10531},{"className":10530,"style":1454},[264],[10532,10548],{"type":150,"tag":169,"props":10533,"children":10534},{"style":2898},[10535,10539],{"type":150,"tag":169,"props":10536,"children":10538},{"className":10537,"style":517},[274],[],{"type":150,"tag":169,"props":10540,"children":10542},{"className":10541},[522,523,524,525],[10543],{"type":150,"tag":169,"props":10544,"children":10546},{"className":10545},[248,489,525],[10547],{"type":155,"value":739},{"type":150,"tag":169,"props":10549,"children":10550},{"style":1475},[10551,10555],{"type":150,"tag":169,"props":10552,"children":10554},{"className":10553,"style":517},[274],[],{"type":150,"tag":169,"props":10556,"children":10558},{"className":10557},[522,523,524,525],[10559],{"type":150,"tag":169,"props":10560,"children":10562},{"className":10561},[248,525],[10563,10568,10573],{"type":150,"tag":169,"props":10564,"children":10566},{"className":10565},[1004,525],[10567],{"type":155,"value":972},{"type":150,"tag":169,"props":10569,"children":10571},{"className":10570},[248,525],[10572],{"type":155,"value":415},{"type":150,"tag":169,"props":10574,"children":10576},{"className":10575},[1015,525],[10577],{"type":155,"value":982},{"type":150,"tag":169,"props":10579,"children":10581},{"className":10580},[355],[10582],{"type":155,"value":358},{"type":150,"tag":169,"props":10584,"children":10586},{"className":10585},[259],[10587],{"type":150,"tag":169,"props":10588,"children":10590},{"className":10589,"style":2960},[264],[10591],{"type":150,"tag":169,"props":10592,"children":10593},{},[],{"type":150,"tag":169,"props":10595,"children":10597},{"className":10596},[552],[10598],{"type":155,"value":421},{"type":150,"tag":169,"props":10600,"children":10602},{"className":10601,"style":559},[558],[],{"type":150,"tag":169,"props":10604,"children":10606},{"className":10605},[630],[10607],{"type":155,"value":442},{"type":150,"tag":169,"props":10609,"children":10611},{"className":10610,"style":559},[558],[],{"type":150,"tag":169,"props":10613,"children":10615},{"className":10614,"style":559},[558],[],{"type":150,"tag":169,"props":10617,"children":10619},{"className":10618},[552],[10620],{"type":155,"value":421},{"type":150,"tag":169,"props":10622,"children":10624},{"className":10623,"style":559},[558],[],{"type":150,"tag":169,"props":10626,"children":10628},{"className":10627},[248],[10629,10634],{"type":150,"tag":169,"props":10630,"children":10632},{"className":10631,"style":2427},[248,489],[10633],{"type":155,"value":1876},{"type":150,"tag":169,"props":10635,"children":10637},{"className":10636},[495],[10638],{"type":150,"tag":169,"props":10639,"children":10641},{"className":10640},[253,254],[10642,10701],{"type":150,"tag":169,"props":10643,"children":10645},{"className":10644},[259],[10646,10696],{"type":150,"tag":169,"props":10647,"children":10649},{"className":10648,"style":1454},[264],[10650,10666],{"type":150,"tag":169,"props":10651,"children":10652},{"style":2898},[10653,10657],{"type":150,"tag":169,"props":10654,"children":10656},{"className":10655,"style":517},[274],[],{"type":150,"tag":169,"props":10658,"children":10660},{"className":10659},[522,523,524,525],[10661],{"type":150,"tag":169,"props":10662,"children":10664},{"className":10663},[248,489,525],[10665],{"type":155,"value":739},{"type":150,"tag":169,"props":10667,"children":10668},{"style":1475},[10669,10673],{"type":150,"tag":169,"props":10670,"children":10672},{"className":10671,"style":517},[274],[],{"type":150,"tag":169,"props":10674,"children":10676},{"className":10675},[522,523,524,525],[10677],{"type":150,"tag":169,"props":10678,"children":10680},{"className":10679},[248,525],[10681,10686,10691],{"type":150,"tag":169,"props":10682,"children":10684},{"className":10683},[1004,525],[10685],{"type":155,"value":972},{"type":150,"tag":169,"props":10687,"children":10689},{"className":10688,"style":2329},[248,489,525],[10690],{"type":155,"value":8100},{"type":150,"tag":169,"props":10692,"children":10694},{"className":10693},[1015,525],[10695],{"type":155,"value":982},{"type":150,"tag":169,"props":10697,"children":10699},{"className":10698},[355],[10700],{"type":155,"value":358},{"type":150,"tag":169,"props":10702,"children":10704},{"className":10703},[259],[10705],{"type":150,"tag":169,"props":10706,"children":10708},{"className":10707,"style":2960},[264],[10709],{"type":150,"tag":169,"props":10710,"children":10711},{},[],{"type":150,"tag":169,"props":10713,"children":10715},{"className":10714},[552],[10716],{"type":155,"value":421},{"type":150,"tag":169,"props":10718,"children":10720},{"className":10719,"style":559},[558],[],{"type":150,"tag":169,"props":10722,"children":10724},{"className":10723},[248],[10725,10730],{"type":150,"tag":169,"props":10726,"children":10728},{"className":10727,"style":2329},[248,489],[10729],{"type":155,"value":1915},{"type":150,"tag":169,"props":10731,"children":10733},{"className":10732},[495],[10734],{"type":150,"tag":169,"props":10735,"children":10737},{"className":10736},[253,254],[10738,10797],{"type":150,"tag":169,"props":10739,"children":10741},{"className":10740},[259],[10742,10792],{"type":150,"tag":169,"props":10743,"children":10745},{"className":10744,"style":1454},[264],[10746,10762],{"type":150,"tag":169,"props":10747,"children":10748},{"style":3292},[10749,10753],{"type":150,"tag":169,"props":10750,"children":10752},{"className":10751,"style":517},[274],[],{"type":150,"tag":169,"props":10754,"children":10756},{"className":10755},[522,523,524,525],[10757],{"type":150,"tag":169,"props":10758,"children":10760},{"className":10759},[248,489,525],[10761],{"type":155,"value":739},{"type":150,"tag":169,"props":10763,"children":10764},{"style":1475},[10765,10769],{"type":150,"tag":169,"props":10766,"children":10768},{"className":10767,"style":517},[274],[],{"type":150,"tag":169,"props":10770,"children":10772},{"className":10771},[522,523,524,525],[10773],{"type":150,"tag":169,"props":10774,"children":10776},{"className":10775},[248,525],[10777,10782,10787],{"type":150,"tag":169,"props":10778,"children":10780},{"className":10779},[1004,525],[10781],{"type":155,"value":972},{"type":150,"tag":169,"props":10783,"children":10785},{"className":10784},[248,525],[10786],{"type":155,"value":415},{"type":150,"tag":169,"props":10788,"children":10790},{"className":10789},[1015,525],[10791],{"type":155,"value":982},{"type":150,"tag":169,"props":10793,"children":10795},{"className":10794},[355],[10796],{"type":155,"value":358},{"type":150,"tag":169,"props":10798,"children":10800},{"className":10799},[259],[10801],{"type":150,"tag":169,"props":10802,"children":10804},{"className":10803,"style":2960},[264],[10805],{"type":150,"tag":169,"props":10806,"children":10807},{},[],{"type":150,"tag":169,"props":10809,"children":10811},{"className":10810},[552],[10812],{"type":155,"value":421},{"type":150,"tag":169,"props":10814,"children":10816},{"className":10815,"style":559},[558],[],{"type":150,"tag":169,"props":10818,"children":10820},{"className":10819},[630],[10821],{"type":155,"value":442},{"type":150,"tag":169,"props":10823,"children":10825},{"className":10824,"style":559},[558],[],{"type":150,"tag":169,"props":10827,"children":10829},{"className":10828,"style":559},[558],[],{"type":150,"tag":169,"props":10831,"children":10833},{"className":10832},[552],[10834],{"type":155,"value":421},{"type":150,"tag":169,"props":10836,"children":10838},{"className":10837,"style":559},[558],[],{"type":150,"tag":169,"props":10840,"children":10842},{"className":10841},[248],[10843,10848],{"type":150,"tag":169,"props":10844,"children":10846},{"className":10845,"style":2329},[248,489],[10847],{"type":155,"value":1915},{"type":150,"tag":169,"props":10849,"children":10851},{"className":10850},[495],[10852],{"type":150,"tag":169,"props":10853,"children":10855},{"className":10854},[253,254],[10856,10915],{"type":150,"tag":169,"props":10857,"children":10859},{"className":10858},[259],[10860,10910],{"type":150,"tag":169,"props":10861,"children":10863},{"className":10862,"style":1454},[264],[10864,10880],{"type":150,"tag":169,"props":10865,"children":10866},{"style":3292},[10867,10871],{"type":150,"tag":169,"props":10868,"children":10870},{"className":10869,"style":517},[274],[],{"type":150,"tag":169,"props":10872,"children":10874},{"className":10873},[522,523,524,525],[10875],{"type":150,"tag":169,"props":10876,"children":10878},{"className":10877},[248,489,525],[10879],{"type":155,"value":739},{"type":150,"tag":169,"props":10881,"children":10882},{"style":1475},[10883,10887],{"type":150,"tag":169,"props":10884,"children":10886},{"className":10885,"style":517},[274],[],{"type":150,"tag":169,"props":10888,"children":10890},{"className":10889},[522,523,524,525],[10891],{"type":150,"tag":169,"props":10892,"children":10894},{"className":10893},[248,525],[10895,10900,10905],{"type":150,"tag":169,"props":10896,"children":10898},{"className":10897},[1004,525],[10899],{"type":155,"value":972},{"type":150,"tag":169,"props":10901,"children":10903},{"className":10902,"style":2329},[248,489,525],[10904],{"type":155,"value":8100},{"type":150,"tag":169,"props":10906,"children":10908},{"className":10907},[1015,525],[10909],{"type":155,"value":982},{"type":150,"tag":169,"props":10911,"children":10913},{"className":10912},[355],[10914],{"type":155,"value":358},{"type":150,"tag":169,"props":10916,"children":10918},{"className":10917},[259],[10919],{"type":150,"tag":169,"props":10920,"children":10922},{"className":10921,"style":2960},[264],[10923],{"type":150,"tag":169,"props":10924,"children":10925},{},[],{"type":150,"tag":169,"props":10927,"children":10929},{"className":10928},[1015],[10930],{"type":155,"value":1326},{"type":150,"tag":169,"props":10932,"children":10934},{"className":10933,"style":839},[558],[],{"type":150,"tag":169,"props":10936,"children":10938},{"className":10937},[844],[10939],{"type":155,"value":744},{"type":150,"tag":169,"props":10941,"children":10943},{"className":10942,"style":839},[558],[],{"type":150,"tag":169,"props":10945,"children":10947},{"className":10946},[237],[10948,10952],{"type":150,"tag":169,"props":10949,"children":10951},{"className":10950,"style":3887},[242],[],{"type":150,"tag":169,"props":10953,"children":10955},{"className":10954},[248],[10956,10961],{"type":150,"tag":169,"props":10957,"children":10959},{"className":10958},[248,867],[10960],{"type":155,"value":754},{"type":150,"tag":169,"props":10962,"children":10964},{"className":10963},[495],[10965],{"type":150,"tag":169,"props":10966,"children":10968},{"className":10967},[253],[10969],{"type":150,"tag":169,"props":10970,"children":10972},{"className":10971},[259],[10973],{"type":150,"tag":169,"props":10974,"children":10976},{"className":10975,"style":3887},[264],[10977],{"type":150,"tag":169,"props":10978,"children":10979},{"style":3908},[10980,10984],{"type":150,"tag":169,"props":10981,"children":10983},{"className":10982,"style":517},[274],[],{"type":150,"tag":169,"props":10985,"children":10987},{"className":10986},[522,523,524,525],[10988],{"type":150,"tag":169,"props":10989,"children":10991},{"className":10990},[248,525],[10992,10997,11002,11059,11064,11121],{"type":150,"tag":169,"props":10993,"children":10995},{"className":10994,"style":2329},[248,489,525],[10996],{"type":155,"value":8100},{"type":150,"tag":169,"props":10998,"children":11000},{"className":10999},[1004,525],[11001],{"type":155,"value":972},{"type":150,"tag":169,"props":11003,"children":11005},{"className":11004},[248,525],[11006,11011],{"type":150,"tag":169,"props":11007,"children":11009},{"className":11008},[248,489,525],[11010],{"type":155,"value":759},{"type":150,"tag":169,"props":11012,"children":11014},{"className":11013},[495],[11015],{"type":150,"tag":169,"props":11016,"children":11018},{"className":11017},[253,254],[11019,11048],{"type":150,"tag":169,"props":11020,"children":11022},{"className":11021},[259],[11023,11043],{"type":150,"tag":169,"props":11024,"children":11026},{"className":11025,"style":4040},[264],[11027],{"type":150,"tag":169,"props":11028,"children":11029},{"style":4044},[11030,11034],{"type":150,"tag":169,"props":11031,"children":11033},{"className":11032,"style":4049},[274],[],{"type":150,"tag":169,"props":11035,"children":11037},{"className":11036},[522,4054,4055,525],[11038],{"type":150,"tag":169,"props":11039,"children":11041},{"className":11040,"style":2427},[248,489,525],[11042],{"type":155,"value":1876},{"type":150,"tag":169,"props":11044,"children":11046},{"className":11045},[355],[11047],{"type":155,"value":358},{"type":150,"tag":169,"props":11049,"children":11051},{"className":11050},[259],[11052],{"type":150,"tag":169,"props":11053,"children":11055},{"className":11054,"style":4074},[264],[11056],{"type":150,"tag":169,"props":11057,"children":11058},{},[],{"type":150,"tag":169,"props":11060,"children":11062},{"className":11061},[5172,525],[11063],{"type":155,"value":5136},{"type":150,"tag":169,"props":11065,"children":11067},{"className":11066},[248,525],[11068,11073],{"type":150,"tag":169,"props":11069,"children":11071},{"className":11070},[248,489,525],[11072],{"type":155,"value":759},{"type":150,"tag":169,"props":11074,"children":11076},{"className":11075},[495],[11077],{"type":150,"tag":169,"props":11078,"children":11080},{"className":11079},[253,254],[11081,11110],{"type":150,"tag":169,"props":11082,"children":11084},{"className":11083},[259],[11085,11105],{"type":150,"tag":169,"props":11086,"children":11088},{"className":11087,"style":4818},[264],[11089],{"type":150,"tag":169,"props":11090,"children":11091},{"style":4822},[11092,11096],{"type":150,"tag":169,"props":11093,"children":11095},{"className":11094,"style":4049},[274],[],{"type":150,"tag":169,"props":11097,"children":11099},{"className":11098},[522,4054,4055,525],[11100],{"type":150,"tag":169,"props":11101,"children":11103},{"className":11102,"style":2329},[248,489,525],[11104],{"type":155,"value":1915},{"type":150,"tag":169,"props":11106,"children":11108},{"className":11107},[355],[11109],{"type":155,"value":358},{"type":150,"tag":169,"props":11111,"children":11113},{"className":11112},[259],[11114],{"type":150,"tag":169,"props":11115,"children":11117},{"className":11116,"style":4849},[264],[11118],{"type":150,"tag":169,"props":11119,"children":11120},{},[],{"type":150,"tag":169,"props":11122,"children":11124},{"className":11123},[1015,525],[11125],{"type":155,"value":982},{"type":150,"tag":151,"props":11127,"children":11128},{},[11129,11131,11233,11235,11338],{"type":155,"value":11130},"而它整个是由 ",{"type":150,"tag":169,"props":11132,"children":11134},{"className":11133},[177],[11135,11164],{"type":150,"tag":169,"props":11136,"children":11138},{"className":11137},[182],[11139],{"type":150,"tag":185,"props":11140,"children":11141},{"xmlns":187},[11142],{"type":150,"tag":191,"props":11143,"children":11144},{},[11145,11159],{"type":150,"tag":195,"props":11146,"children":11147},{},[11148],{"type":150,"tag":401,"props":11149,"children":11150},{},[11151,11155],{"type":150,"tag":405,"props":11152,"children":11153},{},[11154],{"type":155,"value":409},{"type":150,"tag":405,"props":11156,"children":11157},{},[11158],{"type":155,"value":739},{"type":150,"tag":223,"props":11160,"children":11161},{"encoding":225},[11162],{"type":155,"value":11163},"x_i",{"type":150,"tag":169,"props":11165,"children":11167},{"className":11166,"ariaHidden":214},[232],[11168],{"type":150,"tag":169,"props":11169,"children":11171},{"className":11170},[237],[11172,11176],{"type":150,"tag":169,"props":11173,"children":11175},{"className":11174,"style":1343},[242],[],{"type":150,"tag":169,"props":11177,"children":11179},{"className":11178},[248],[11180,11185],{"type":150,"tag":169,"props":11181,"children":11183},{"className":11182},[248,489],[11184],{"type":155,"value":409},{"type":150,"tag":169,"props":11186,"children":11188},{"className":11187},[495],[11189],{"type":150,"tag":169,"props":11190,"children":11192},{"className":11191},[253,254],[11193,11222],{"type":150,"tag":169,"props":11194,"children":11196},{"className":11195},[259],[11197,11217],{"type":150,"tag":169,"props":11198,"children":11200},{"className":11199,"style":802},[264],[11201],{"type":150,"tag":169,"props":11202,"children":11203},{"style":512},[11204,11208],{"type":150,"tag":169,"props":11205,"children":11207},{"className":11206,"style":517},[274],[],{"type":150,"tag":169,"props":11209,"children":11211},{"className":11210},[522,523,524,525],[11212],{"type":150,"tag":169,"props":11213,"children":11215},{"className":11214},[248,489,525],[11216],{"type":155,"value":739},{"type":150,"tag":169,"props":11218,"children":11220},{"className":11219},[355],[11221],{"type":155,"value":358},{"type":150,"tag":169,"props":11223,"children":11225},{"className":11224},[259],[11226],{"type":150,"tag":169,"props":11227,"children":11229},{"className":11228,"style":544},[264],[11230],{"type":150,"tag":169,"props":11231,"children":11232},{},[],{"type":155,"value":11234}," 一次矩阵乘法得到的。把各组投影矩阵横向拼起来记作 ",{"type":150,"tag":169,"props":11236,"children":11238},{"className":11237},[177],[11239,11268],{"type":150,"tag":169,"props":11240,"children":11242},{"className":11241},[182],[11243],{"type":150,"tag":185,"props":11244,"children":11245},{"xmlns":187},[11246],{"type":150,"tag":191,"props":11247,"children":11248},{},[11249,11263],{"type":150,"tag":195,"props":11250,"children":11251},{},[11252],{"type":150,"tag":401,"props":11253,"children":11254},{},[11255,11259],{"type":150,"tag":405,"props":11256,"children":11257},{},[11258],{"type":155,"value":3443},{"type":150,"tag":405,"props":11260,"children":11261},{},[11262],{"type":155,"value":10046},{"type":150,"tag":223,"props":11264,"children":11265},{"encoding":225},[11266],{"type":155,"value":11267},"W_c",{"type":150,"tag":169,"props":11269,"children":11271},{"className":11270,"ariaHidden":214},[232],[11272],{"type":150,"tag":169,"props":11273,"children":11275},{"className":11274},[237],[11276,11280],{"type":150,"tag":169,"props":11277,"children":11279},{"className":11278,"style":6364},[242],[],{"type":150,"tag":169,"props":11281,"children":11283},{"className":11282},[248],[11284,11289],{"type":150,"tag":169,"props":11285,"children":11287},{"className":11286,"style":3869},[248,489],[11288],{"type":155,"value":3443},{"type":150,"tag":169,"props":11290,"children":11292},{"className":11291},[495],[11293],{"type":150,"tag":169,"props":11294,"children":11296},{"className":11295},[253,254],[11297,11327],{"type":150,"tag":169,"props":11298,"children":11300},{"className":11299},[259],[11301,11322],{"type":150,"tag":169,"props":11302,"children":11304},{"className":11303,"style":5039},[264],[11305],{"type":150,"tag":169,"props":11306,"children":11308},{"style":11307},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[11309,11313],{"type":150,"tag":169,"props":11310,"children":11312},{"className":11311,"style":517},[274],[],{"type":150,"tag":169,"props":11314,"children":11316},{"className":11315},[522,523,524,525],[11317],{"type":150,"tag":169,"props":11318,"children":11320},{"className":11319},[248,489,525],[11321],{"type":155,"value":10046},{"type":150,"tag":169,"props":11323,"children":11325},{"className":11324},[355],[11326],{"type":155,"value":358},{"type":150,"tag":169,"props":11328,"children":11330},{"className":11329},[259],[11331],{"type":150,"tag":169,"props":11332,"children":11334},{"className":11333,"style":544},[264],[11335],{"type":150,"tag":169,"props":11336,"children":11337},{},[],{"type":155,"value":11339},"：",{"type":150,"tag":169,"props":11341,"children":11343},{"className":11342},[172],[11344],{"type":150,"tag":169,"props":11345,"children":11347},{"className":11346},[177],[11348,11634],{"type":150,"tag":169,"props":11349,"children":11351},{"className":11350},[182],[11352],{"type":150,"tag":185,"props":11353,"children":11354},{"xmlns":187,"display":188},[11355],{"type":150,"tag":191,"props":11356,"children":11357},{},[11358,11629],{"type":150,"tag":195,"props":11359,"children":11360},{},[11361,11372,11376,11387,11398,11402,11406,11417,11421,11425,11451,11455,11459,11463,11467,11493,11497,11523,11527,11531,11535,11539,11565,11569,11573],{"type":150,"tag":401,"props":11362,"children":11363},{},[11364,11368],{"type":150,"tag":405,"props":11365,"children":11366},{},[11367],{"type":155,"value":10046},{"type":150,"tag":405,"props":11369,"children":11370},{},[11371],{"type":155,"value":739},{"type":150,"tag":417,"props":11373,"children":11374},{},[11375],{"type":155,"value":1045},{"type":150,"tag":401,"props":11377,"children":11378},{},[11379,11383],{"type":150,"tag":405,"props":11380,"children":11381},{},[11382],{"type":155,"value":409},{"type":150,"tag":405,"props":11384,"children":11385},{},[11386],{"type":155,"value":739},{"type":150,"tag":401,"props":11388,"children":11389},{},[11390,11394],{"type":150,"tag":405,"props":11391,"children":11392},{},[11393],{"type":155,"value":3443},{"type":150,"tag":405,"props":11395,"children":11396},{},[11397],{"type":155,"value":10046},{"type":150,"tag":417,"props":11399,"children":11400},{"separator":214},[11401],{"type":155,"value":421},{"type":150,"tag":558,"props":11403,"children":11405},{"width":11404},"2em",[],{"type":150,"tag":401,"props":11407,"children":11408},{},[11409,11413],{"type":150,"tag":405,"props":11410,"children":11411},{},[11412],{"type":155,"value":3443},{"type":150,"tag":405,"props":11414,"children":11415},{},[11416],{"type":155,"value":10046},{"type":150,"tag":417,"props":11418,"children":11419},{},[11420],{"type":155,"value":1045},{"type":150,"tag":417,"props":11422,"children":11423},{"stretchy":207},[11424],{"type":155,"value":1222},{"type":150,"tag":1224,"props":11426,"children":11427},{},[11428,11432,11436],{"type":150,"tag":405,"props":11429,"children":11430},{},[11431],{"type":155,"value":3443},{"type":150,"tag":405,"props":11433,"children":11434},{},[11435],{"type":155,"value":1876},{"type":150,"tag":195,"props":11437,"children":11438},{},[11439,11443,11447],{"type":150,"tag":417,"props":11440,"children":11441},{"stretchy":207},[11442],{"type":155,"value":972},{"type":150,"tag":411,"props":11444,"children":11445},{},[11446],{"type":155,"value":415},{"type":150,"tag":417,"props":11448,"children":11449},{"stretchy":207},[11450],{"type":155,"value":982},{"type":150,"tag":417,"props":11452,"children":11453},{"separator":214},[11454],{"type":155,"value":421},{"type":150,"tag":417,"props":11456,"children":11457},{},[11458],{"type":155,"value":442},{"type":150,"tag":217,"props":11460,"children":11461},{},[11462],{"type":155,"value":447},{"type":150,"tag":417,"props":11464,"children":11465},{"separator":214},[11466],{"type":155,"value":421},{"type":150,"tag":1224,"props":11468,"children":11469},{},[11470,11474,11478],{"type":150,"tag":405,"props":11471,"children":11472},{},[11473],{"type":155,"value":3443},{"type":150,"tag":405,"props":11475,"children":11476},{},[11477],{"type":155,"value":1876},{"type":150,"tag":195,"props":11479,"children":11480},{},[11481,11485,11489],{"type":150,"tag":417,"props":11482,"children":11483},{"stretchy":207},[11484],{"type":155,"value":972},{"type":150,"tag":405,"props":11486,"children":11487},{},[11488],{"type":155,"value":8100},{"type":150,"tag":417,"props":11490,"children":11491},{"stretchy":207},[11492],{"type":155,"value":982},{"type":150,"tag":417,"props":11494,"children":11495},{"separator":214},[11496],{"type":155,"value":421},{"type":150,"tag":1224,"props":11498,"children":11499},{},[11500,11504,11508],{"type":150,"tag":405,"props":11501,"children":11502},{},[11503],{"type":155,"value":3443},{"type":150,"tag":405,"props":11505,"children":11506},{},[11507],{"type":155,"value":1915},{"type":150,"tag":195,"props":11509,"children":11510},{},[11511,11515,11519],{"type":150,"tag":417,"props":11512,"children":11513},{"stretchy":207},[11514],{"type":155,"value":972},{"type":150,"tag":411,"props":11516,"children":11517},{},[11518],{"type":155,"value":415},{"type":150,"tag":417,"props":11520,"children":11521},{"stretchy":207},[11522],{"type":155,"value":982},{"type":150,"tag":417,"props":11524,"children":11525},{"separator":214},[11526],{"type":155,"value":421},{"type":150,"tag":417,"props":11528,"children":11529},{},[11530],{"type":155,"value":442},{"type":150,"tag":217,"props":11532,"children":11533},{},[11534],{"type":155,"value":447},{"type":150,"tag":417,"props":11536,"children":11537},{"separator":214},[11538],{"type":155,"value":421},{"type":150,"tag":1224,"props":11540,"children":11541},{},[11542,11546,11550],{"type":150,"tag":405,"props":11543,"children":11544},{},[11545],{"type":155,"value":3443},{"type":150,"tag":405,"props":11547,"children":11548},{},[11549],{"type":155,"value":1915},{"type":150,"tag":195,"props":11551,"children":11552},{},[11553,11557,11561],{"type":150,"tag":417,"props":11554,"children":11555},{"stretchy":207},[11556],{"type":155,"value":972},{"type":150,"tag":405,"props":11558,"children":11559},{},[11560],{"type":155,"value":8100},{"type":150,"tag":417,"props":11562,"children":11563},{"stretchy":207},[11564],{"type":155,"value":982},{"type":150,"tag":417,"props":11566,"children":11567},{"stretchy":207},[11568],{"type":155,"value":1326},{"type":150,"tag":417,"props":11570,"children":11571},{},[11572],{"type":155,"value":744},{"type":150,"tag":746,"props":11574,"children":11575},{},[11576,11580],{"type":150,"tag":405,"props":11577,"children":11578},{"mathvariant":751},[11579],{"type":155,"value":754},{"type":150,"tag":195,"props":11581,"children":11582},{},[11583,11587,11591,11595,11599,11610,11614,11625],{"type":150,"tag":405,"props":11584,"children":11585},{},[11586],{"type":155,"value":759},{"type":150,"tag":417,"props":11588,"children":11589},{},[11590],{"type":155,"value":5554},{"type":150,"tag":405,"props":11592,"children":11593},{},[11594],{"type":155,"value":8100},{"type":150,"tag":417,"props":11596,"children":11597},{"stretchy":207},[11598],{"type":155,"value":972},{"type":150,"tag":401,"props":11600,"children":11601},{},[11602,11606],{"type":150,"tag":405,"props":11603,"children":11604},{},[11605],{"type":155,"value":759},{"type":150,"tag":405,"props":11607,"children":11608},{},[11609],{"type":155,"value":1876},{"type":150,"tag":417,"props":11611,"children":11612},{},[11613],{"type":155,"value":5136},{"type":150,"tag":401,"props":11615,"children":11616},{},[11617,11621],{"type":150,"tag":405,"props":11618,"children":11619},{},[11620],{"type":155,"value":759},{"type":150,"tag":405,"props":11622,"children":11623},{},[11624],{"type":155,"value":1915},{"type":150,"tag":417,"props":11626,"children":11627},{"stretchy":207},[11628],{"type":155,"value":982},{"type":150,"tag":223,"props":11630,"children":11631},{"encoding":225},[11632],{"type":155,"value":11633},"c_i = x_i W_c, \\qquad W_c = [W_k^{(1)}, \\cdots, W_k^{(g)}, W_v^{(1)}, \\cdots, W_v^{(g)}] \\in \\mathbb{R}^{d \\times g(d_k + d_v)}",{"type":150,"tag":169,"props":11635,"children":11637},{"className":11636,"ariaHidden":214},[232],[11638,11716,11923,12373],{"type":150,"tag":169,"props":11639,"children":11641},{"className":11640},[237],[11642,11646,11703,11707,11712],{"type":150,"tag":169,"props":11643,"children":11645},{"className":11644,"style":1343},[242],[],{"type":150,"tag":169,"props":11647,"children":11649},{"className":11648},[248],[11650,11655],{"type":150,"tag":169,"props":11651,"children":11653},{"className":11652},[248,489],[11654],{"type":155,"value":10046},{"type":150,"tag":169,"props":11656,"children":11658},{"className":11657},[495],[11659],{"type":150,"tag":169,"props":11660,"children":11662},{"className":11661},[253,254],[11663,11692],{"type":150,"tag":169,"props":11664,"children":11666},{"className":11665},[259],[11667,11687],{"type":150,"tag":169,"props":11668,"children":11670},{"className":11669,"style":802},[264],[11671],{"type":150,"tag":169,"props":11672,"children":11673},{"style":512},[11674,11678],{"type":150,"tag":169,"props":11675,"children":11677},{"className":11676,"style":517},[274],[],{"type":150,"tag":169,"props":11679,"children":11681},{"className":11680},[522,523,524,525],[11682],{"type":150,"tag":169,"props":11683,"children":11685},{"className":11684},[248,489,525],[11686],{"type":155,"value":739},{"type":150,"tag":169,"props":11688,"children":11690},{"className":11689},[355],[11691],{"type":155,"value":358},{"type":150,"tag":169,"props":11693,"children":11695},{"className":11694},[259],[11696],{"type":150,"tag":169,"props":11697,"children":11699},{"className":11698,"style":544},[264],[11700],{"type":150,"tag":169,"props":11701,"children":11702},{},[],{"type":150,"tag":169,"props":11704,"children":11706},{"className":11705,"style":839},[558],[],{"type":150,"tag":169,"props":11708,"children":11710},{"className":11709},[844],[11711],{"type":155,"value":1045},{"type":150,"tag":169,"props":11713,"children":11715},{"className":11714,"style":839},[558],[],{"type":150,"tag":169,"props":11717,"children":11719},{"className":11718},[237],[11720,11725,11782,11839,11844,11849,11853,11910,11914,11919],{"type":150,"tag":169,"props":11721,"children":11724},{"className":11722,"style":11723},[242],"height:0.8778em;vertical-align:-0.1944em;",[],{"type":150,"tag":169,"props":11726,"children":11728},{"className":11727},[248],[11729,11734],{"type":150,"tag":169,"props":11730,"children":11732},{"className":11731},[248,489],[11733],{"type":155,"value":409},{"type":150,"tag":169,"props":11735,"children":11737},{"className":11736},[495],[11738],{"type":150,"tag":169,"props":11739,"children":11741},{"className":11740},[253,254],[11742,11771],{"type":150,"tag":169,"props":11743,"children":11745},{"className":11744},[259],[11746,11766],{"type":150,"tag":169,"props":11747,"children":11749},{"className":11748,"style":802},[264],[11750],{"type":150,"tag":169,"props":11751,"children":11752},{"style":512},[11753,11757],{"type":150,"tag":169,"props":11754,"children":11756},{"className":11755,"style":517},[274],[],{"type":150,"tag":169,"props":11758,"children":11760},{"className":11759},[522,523,524,525],[11761],{"type":150,"tag":169,"props":11762,"children":11764},{"className":11763},[248,489,525],[11765],{"type":155,"value":739},{"type":150,"tag":169,"props":11767,"children":11769},{"className":11768},[355],[11770],{"type":155,"value":358},{"type":150,"tag":169,"props":11772,"children":11774},{"className":11773},[259],[11775],{"type":150,"tag":169,"props":11776,"children":11778},{"className":11777,"style":544},[264],[11779],{"type":150,"tag":169,"props":11780,"children":11781},{},[],{"type":150,"tag":169,"props":11783,"children":11785},{"className":11784},[248],[11786,11791],{"type":150,"tag":169,"props":11787,"children":11789},{"className":11788,"style":3869},[248,489],[11790],{"type":155,"value":3443},{"type":150,"tag":169,"props":11792,"children":11794},{"className":11793},[495],[11795],{"type":150,"tag":169,"props":11796,"children":11798},{"className":11797},[253,254],[11799,11828],{"type":150,"tag":169,"props":11800,"children":11802},{"className":11801},[259],[11803,11823],{"type":150,"tag":169,"props":11804,"children":11806},{"className":11805,"style":5039},[264],[11807],{"type":150,"tag":169,"props":11808,"children":11809},{"style":11307},[11810,11814],{"type":150,"tag":169,"props":11811,"children":11813},{"className":11812,"style":517},[274],[],{"type":150,"tag":169,"props":11815,"children":11817},{"className":11816},[522,523,524,525],[11818],{"type":150,"tag":169,"props":11819,"children":11821},{"className":11820},[248,489,525],[11822],{"type":155,"value":10046},{"type":150,"tag":169,"props":11824,"children":11826},{"className":11825},[355],[11827],{"type":155,"value":358},{"type":150,"tag":169,"props":11829,"children":11831},{"className":11830},[259],[11832],{"type":150,"tag":169,"props":11833,"children":11835},{"className":11834,"style":544},[264],[11836],{"type":150,"tag":169,"props":11837,"children":11838},{},[],{"type":150,"tag":169,"props":11840,"children":11842},{"className":11841},[552],[11843],{"type":155,"value":421},{"type":150,"tag":169,"props":11845,"children":11848},{"className":11846,"style":11847},[558],"margin-right:2em;",[],{"type":150,"tag":169,"props":11850,"children":11852},{"className":11851,"style":559},[558],[],{"type":150,"tag":169,"props":11854,"children":11856},{"className":11855},[248],[11857,11862],{"type":150,"tag":169,"props":11858,"children":11860},{"className":11859,"style":3869},[248,489],[11861],{"type":155,"value":3443},{"type":150,"tag":169,"props":11863,"children":11865},{"className":11864},[495],[11866],{"type":150,"tag":169,"props":11867,"children":11869},{"className":11868},[253,254],[11870,11899],{"type":150,"tag":169,"props":11871,"children":11873},{"className":11872},[259],[11874,11894],{"type":150,"tag":169,"props":11875,"children":11877},{"className":11876,"style":5039},[264],[11878],{"type":150,"tag":169,"props":11879,"children":11880},{"style":11307},[11881,11885],{"type":150,"tag":169,"props":11882,"children":11884},{"className":11883,"style":517},[274],[],{"type":150,"tag":169,"props":11886,"children":11888},{"className":11887},[522,523,524,525],[11889],{"type":150,"tag":169,"props":11890,"children":11892},{"className":11891},[248,489,525],[11893],{"type":155,"value":10046},{"type":150,"tag":169,"props":11895,"children":11897},{"className":11896},[355],[11898],{"type":155,"value":358},{"type":150,"tag":169,"props":11900,"children":11902},{"className":11901},[259],[11903],{"type":150,"tag":169,"props":11904,"children":11906},{"className":11905,"style":544},[264],[11907],{"type":150,"tag":169,"props":11908,"children":11909},{},[],{"type":150,"tag":169,"props":11911,"children":11913},{"className":11912,"style":839},[558],[],{"type":150,"tag":169,"props":11915,"children":11917},{"className":11916},[844],[11918],{"type":155,"value":1045},{"type":150,"tag":169,"props":11920,"children":11922},{"className":11921,"style":839},[558],[],{"type":150,"tag":169,"props":11924,"children":11926},{"className":11925},[237],[11927,11931,11936,12023,12028,12032,12037,12041,12045,12050,12054,12141,12146,12150,12237,12242,12246,12251,12255,12259,12264,12268,12355,12360,12364,12369],{"type":150,"tag":169,"props":11928,"children":11930},{"className":11929,"style":4200},[242],[],{"type":150,"tag":169,"props":11932,"children":11934},{"className":11933},[1004],[11935],{"type":155,"value":1222},{"type":150,"tag":169,"props":11937,"children":11939},{"className":11938},[248],[11940,11945],{"type":150,"tag":169,"props":11941,"children":11943},{"className":11942,"style":3869},[248,489],[11944],{"type":155,"value":3443},{"type":150,"tag":169,"props":11946,"children":11948},{"className":11947},[495],[11949],{"type":150,"tag":169,"props":11950,"children":11952},{"className":11951},[253,254],[11953,12012],{"type":150,"tag":169,"props":11954,"children":11956},{"className":11955},[259],[11957,12007],{"type":150,"tag":169,"props":11958,"children":11960},{"className":11959,"style":1454},[264],[11961,11977],{"type":150,"tag":169,"props":11962,"children":11963},{"style":4286},[11964,11968],{"type":150,"tag":169,"props":11965,"children":11967},{"className":11966,"style":517},[274],[],{"type":150,"tag":169,"props":11969,"children":11971},{"className":11970},[522,523,524,525],[11972],{"type":150,"tag":169,"props":11973,"children":11975},{"className":11974,"style":2427},[248,489,525],[11976],{"type":155,"value":1876},{"type":150,"tag":169,"props":11978,"children":11979},{"style":1475},[11980,11984],{"type":150,"tag":169,"props":11981,"children":11983},{"className":11982,"style":517},[274],[],{"type":150,"tag":169,"props":11985,"children":11987},{"className":11986},[522,523,524,525],[11988],{"type":150,"tag":169,"props":11989,"children":11991},{"className":11990},[248,525],[11992,11997,12002],{"type":150,"tag":169,"props":11993,"children":11995},{"className":11994},[1004,525],[11996],{"type":155,"value":972},{"type":150,"tag":169,"props":11998,"children":12000},{"className":11999},[248,525],[12001],{"type":155,"value":415},{"type":150,"tag":169,"props":12003,"children":12005},{"className":12004},[1015,525],[12006],{"type":155,"value":982},{"type":150,"tag":169,"props":12008,"children":12010},{"className":12009},[355],[12011],{"type":155,"value":358},{"type":150,"tag":169,"props":12013,"children":12015},{"className":12014},[259],[12016],{"type":150,"tag":169,"props":12017,"children":12019},{"className":12018,"style":4343},[264],[12020],{"type":150,"tag":169,"props":12021,"children":12022},{},[],{"type":150,"tag":169,"props":12024,"children":12026},{"className":12025},[552],[12027],{"type":155,"value":421},{"type":150,"tag":169,"props":12029,"children":12031},{"className":12030,"style":559},[558],[],{"type":150,"tag":169,"props":12033,"children":12035},{"className":12034},[630],[12036],{"type":155,"value":442},{"type":150,"tag":169,"props":12038,"children":12040},{"className":12039,"style":559},[558],[],{"type":150,"tag":169,"props":12042,"children":12044},{"className":12043,"style":559},[558],[],{"type":150,"tag":169,"props":12046,"children":12048},{"className":12047},[552],[12049],{"type":155,"value":421},{"type":150,"tag":169,"props":12051,"children":12053},{"className":12052,"style":559},[558],[],{"type":150,"tag":169,"props":12055,"children":12057},{"className":12056},[248],[12058,12063],{"type":150,"tag":169,"props":12059,"children":12061},{"className":12060,"style":3869},[248,489],[12062],{"type":155,"value":3443},{"type":150,"tag":169,"props":12064,"children":12066},{"className":12065},[495],[12067],{"type":150,"tag":169,"props":12068,"children":12070},{"className":12069},[253,254],[12071,12130],{"type":150,"tag":169,"props":12072,"children":12074},{"className":12073},[259],[12075,12125],{"type":150,"tag":169,"props":12076,"children":12078},{"className":12077,"style":1454},[264],[12079,12095],{"type":150,"tag":169,"props":12080,"children":12081},{"style":4286},[12082,12086],{"type":150,"tag":169,"props":12083,"children":12085},{"className":12084,"style":517},[274],[],{"type":150,"tag":169,"props":12087,"children":12089},{"className":12088},[522,523,524,525],[12090],{"type":150,"tag":169,"props":12091,"children":12093},{"className":12092,"style":2427},[248,489,525],[12094],{"type":155,"value":1876},{"type":150,"tag":169,"props":12096,"children":12097},{"style":1475},[12098,12102],{"type":150,"tag":169,"props":12099,"children":12101},{"className":12100,"style":517},[274],[],{"type":150,"tag":169,"props":12103,"children":12105},{"className":12104},[522,523,524,525],[12106],{"type":150,"tag":169,"props":12107,"children":12109},{"className":12108},[248,525],[12110,12115,12120],{"type":150,"tag":169,"props":12111,"children":12113},{"className":12112},[1004,525],[12114],{"type":155,"value":972},{"type":150,"tag":169,"props":12116,"children":12118},{"className":12117,"style":2329},[248,489,525],[12119],{"type":155,"value":8100},{"type":150,"tag":169,"props":12121,"children":12123},{"className":12122},[1015,525],[12124],{"type":155,"value":982},{"type":150,"tag":169,"props":12126,"children":12128},{"className":12127},[355],[12129],{"type":155,"value":358},{"type":150,"tag":169,"props":12131,"children":12133},{"className":12132},[259],[12134],{"type":150,"tag":169,"props":12135,"children":12137},{"className":12136,"style":4343},[264],[12138],{"type":150,"tag":169,"props":12139,"children":12140},{},[],{"type":150,"tag":169,"props":12142,"children":12144},{"className":12143},[552],[12145],{"type":155,"value":421},{"type":150,"tag":169,"props":12147,"children":12149},{"className":12148,"style":559},[558],[],{"type":150,"tag":169,"props":12151,"children":12153},{"className":12152},[248],[12154,12159],{"type":150,"tag":169,"props":12155,"children":12157},{"className":12156,"style":3869},[248,489],[12158],{"type":155,"value":3443},{"type":150,"tag":169,"props":12160,"children":12162},{"className":12161},[495],[12163],{"type":150,"tag":169,"props":12164,"children":12166},{"className":12165},[253,254],[12167,12226],{"type":150,"tag":169,"props":12168,"children":12170},{"className":12169},[259],[12171,12221],{"type":150,"tag":169,"props":12172,"children":12174},{"className":12173,"style":3887},[264],[12175,12191],{"type":150,"tag":169,"props":12176,"children":12177},{"style":3891},[12178,12182],{"type":150,"tag":169,"props":12179,"children":12181},{"className":12180,"style":517},[274],[],{"type":150,"tag":169,"props":12183,"children":12185},{"className":12184},[522,523,524,525],[12186],{"type":150,"tag":169,"props":12187,"children":12189},{"className":12188,"style":2329},[248,489,525],[12190],{"type":155,"value":1915},{"type":150,"tag":169,"props":12192,"children":12193},{"style":3908},[12194,12198],{"type":150,"tag":169,"props":12195,"children":12197},{"className":12196,"style":517},[274],[],{"type":150,"tag":169,"props":12199,"children":12201},{"className":12200},[522,523,524,525],[12202],{"type":150,"tag":169,"props":12203,"children":12205},{"className":12204},[248,525],[12206,12211,12216],{"type":150,"tag":169,"props":12207,"children":12209},{"className":12208},[1004,525],[12210],{"type":155,"value":972},{"type":150,"tag":169,"props":12212,"children":12214},{"className":12213},[248,525],[12215],{"type":155,"value":415},{"type":150,"tag":169,"props":12217,"children":12219},{"className":12218},[1015,525],[12220],{"type":155,"value":982},{"type":150,"tag":169,"props":12222,"children":12224},{"className":12223},[355],[12225],{"type":155,"value":358},{"type":150,"tag":169,"props":12227,"children":12229},{"className":12228},[259],[12230],{"type":150,"tag":169,"props":12231,"children":12233},{"className":12232,"style":4728},[264],[12234],{"type":150,"tag":169,"props":12235,"children":12236},{},[],{"type":150,"tag":169,"props":12238,"children":12240},{"className":12239},[552],[12241],{"type":155,"value":421},{"type":150,"tag":169,"props":12243,"children":12245},{"className":12244,"style":559},[558],[],{"type":150,"tag":169,"props":12247,"children":12249},{"className":12248},[630],[12250],{"type":155,"value":442},{"type":150,"tag":169,"props":12252,"children":12254},{"className":12253,"style":559},[558],[],{"type":150,"tag":169,"props":12256,"children":12258},{"className":12257,"style":559},[558],[],{"type":150,"tag":169,"props":12260,"children":12262},{"className":12261},[552],[12263],{"type":155,"value":421},{"type":150,"tag":169,"props":12265,"children":12267},{"className":12266,"style":559},[558],[],{"type":150,"tag":169,"props":12269,"children":12271},{"className":12270},[248],[12272,12277],{"type":150,"tag":169,"props":12273,"children":12275},{"className":12274,"style":3869},[248,489],[12276],{"type":155,"value":3443},{"type":150,"tag":169,"props":12278,"children":12280},{"className":12279},[495],[12281],{"type":150,"tag":169,"props":12282,"children":12284},{"className":12283},[253,254],[12285,12344],{"type":150,"tag":169,"props":12286,"children":12288},{"className":12287},[259],[12289,12339],{"type":150,"tag":169,"props":12290,"children":12292},{"className":12291,"style":3887},[264],[12293,12309],{"type":150,"tag":169,"props":12294,"children":12295},{"style":3891},[12296,12300],{"type":150,"tag":169,"props":12297,"children":12299},{"className":12298,"style":517},[274],[],{"type":150,"tag":169,"props":12301,"children":12303},{"className":12302},[522,523,524,525],[12304],{"type":150,"tag":169,"props":12305,"children":12307},{"className":12306,"style":2329},[248,489,525],[12308],{"type":155,"value":1915},{"type":150,"tag":169,"props":12310,"children":12311},{"style":3908},[12312,12316],{"type":150,"tag":169,"props":12313,"children":12315},{"className":12314,"style":517},[274],[],{"type":150,"tag":169,"props":12317,"children":12319},{"className":12318},[522,523,524,525],[12320],{"type":150,"tag":169,"props":12321,"children":12323},{"className":12322},[248,525],[12324,12329,12334],{"type":150,"tag":169,"props":12325,"children":12327},{"className":12326},[1004,525],[12328],{"type":155,"value":972},{"type":150,"tag":169,"props":12330,"children":12332},{"className":12331,"style":2329},[248,489,525],[12333],{"type":155,"value":8100},{"type":150,"tag":169,"props":12335,"children":12337},{"className":12336},[1015,525],[12338],{"type":155,"value":982},{"type":150,"tag":169,"props":12340,"children":12342},{"className":12341},[355],[12343],{"type":155,"value":358},{"type":150,"tag":169,"props":12345,"children":12347},{"className":12346},[259],[12348],{"type":150,"tag":169,"props":12349,"children":12351},{"className":12350,"style":4728},[264],[12352],{"type":150,"tag":169,"props":12353,"children":12354},{},[],{"type":150,"tag":169,"props":12356,"children":12358},{"className":12357},[1015],[12359],{"type":155,"value":1326},{"type":150,"tag":169,"props":12361,"children":12363},{"className":12362,"style":839},[558],[],{"type":150,"tag":169,"props":12365,"children":12367},{"className":12366},[844],[12368],{"type":155,"value":744},{"type":150,"tag":169,"props":12370,"children":12372},{"className":12371,"style":839},[558],[],{"type":150,"tag":169,"props":12374,"children":12376},{"className":12375},[237],[12377,12381],{"type":150,"tag":169,"props":12378,"children":12380},{"className":12379,"style":3887},[242],[],{"type":150,"tag":169,"props":12382,"children":12384},{"className":12383},[248],[12385,12390],{"type":150,"tag":169,"props":12386,"children":12388},{"className":12387},[248,867],[12389],{"type":155,"value":754},{"type":150,"tag":169,"props":12391,"children":12393},{"className":12392},[495],[12394],{"type":150,"tag":169,"props":12395,"children":12397},{"className":12396},[253],[12398],{"type":150,"tag":169,"props":12399,"children":12401},{"className":12400},[259],[12402],{"type":150,"tag":169,"props":12403,"children":12405},{"className":12404,"style":3887},[264],[12406],{"type":150,"tag":169,"props":12407,"children":12408},{"style":3908},[12409,12413],{"type":150,"tag":169,"props":12410,"children":12412},{"className":12411,"style":517},[274],[],{"type":150,"tag":169,"props":12414,"children":12416},{"className":12415},[522,523,524,525],[12417],{"type":150,"tag":169,"props":12418,"children":12420},{"className":12419},[248,525],[12421,12426,12431,12436,12441,12498,12503,12560],{"type":150,"tag":169,"props":12422,"children":12424},{"className":12423},[248,489,525],[12425],{"type":155,"value":759},{"type":150,"tag":169,"props":12427,"children":12429},{"className":12428},[5172,525],[12430],{"type":155,"value":5554},{"type":150,"tag":169,"props":12432,"children":12434},{"className":12433,"style":2329},[248,489,525],[12435],{"type":155,"value":8100},{"type":150,"tag":169,"props":12437,"children":12439},{"className":12438},[1004,525],[12440],{"type":155,"value":972},{"type":150,"tag":169,"props":12442,"children":12444},{"className":12443},[248,525],[12445,12450],{"type":150,"tag":169,"props":12446,"children":12448},{"className":12447},[248,489,525],[12449],{"type":155,"value":759},{"type":150,"tag":169,"props":12451,"children":12453},{"className":12452},[495],[12454],{"type":150,"tag":169,"props":12455,"children":12457},{"className":12456},[253,254],[12458,12487],{"type":150,"tag":169,"props":12459,"children":12461},{"className":12460},[259],[12462,12482],{"type":150,"tag":169,"props":12463,"children":12465},{"className":12464,"style":4040},[264],[12466],{"type":150,"tag":169,"props":12467,"children":12468},{"style":4044},[12469,12473],{"type":150,"tag":169,"props":12470,"children":12472},{"className":12471,"style":4049},[274],[],{"type":150,"tag":169,"props":12474,"children":12476},{"className":12475},[522,4054,4055,525],[12477],{"type":150,"tag":169,"props":12478,"children":12480},{"className":12479,"style":2427},[248,489,525],[12481],{"type":155,"value":1876},{"type":150,"tag":169,"props":12483,"children":12485},{"className":12484},[355],[12486],{"type":155,"value":358},{"type":150,"tag":169,"props":12488,"children":12490},{"className":12489},[259],[12491],{"type":150,"tag":169,"props":12492,"children":12494},{"className":12493,"style":4074},[264],[12495],{"type":150,"tag":169,"props":12496,"children":12497},{},[],{"type":150,"tag":169,"props":12499,"children":12501},{"className":12500},[5172,525],[12502],{"type":155,"value":5136},{"type":150,"tag":169,"props":12504,"children":12506},{"className":12505},[248,525],[12507,12512],{"type":150,"tag":169,"props":12508,"children":12510},{"className":12509},[248,489,525],[12511],{"type":155,"value":759},{"type":150,"tag":169,"props":12513,"children":12515},{"className":12514},[495],[12516],{"type":150,"tag":169,"props":12517,"children":12519},{"className":12518},[253,254],[12520,12549],{"type":150,"tag":169,"props":12521,"children":12523},{"className":12522},[259],[12524,12544],{"type":150,"tag":169,"props":12525,"children":12527},{"className":12526,"style":4818},[264],[12528],{"type":150,"tag":169,"props":12529,"children":12530},{"style":4822},[12531,12535],{"type":150,"tag":169,"props":12532,"children":12534},{"className":12533,"style":4049},[274],[],{"type":150,"tag":169,"props":12536,"children":12538},{"className":12537},[522,4054,4055,525],[12539],{"type":150,"tag":169,"props":12540,"children":12542},{"className":12541,"style":2329},[248,489,525],[12543],{"type":155,"value":1915},{"type":150,"tag":169,"props":12545,"children":12547},{"className":12546},[355],[12548],{"type":155,"value":358},{"type":150,"tag":169,"props":12550,"children":12552},{"className":12551},[259],[12553],{"type":150,"tag":169,"props":12554,"children":12556},{"className":12555,"style":4849},[264],[12557],{"type":150,"tag":169,"props":12558,"children":12559},{},[],{"type":150,"tag":169,"props":12561,"children":12563},{"className":12562},[1015,525],[12564],{"type":155,"value":982},{"type":150,"tag":151,"props":12566,"children":12567},{},[12568,12570,12920,12922,13118,13120,13125,13127,13169,13171,13231,13233,13275],{"type":155,"value":12569},"因为 ",{"type":150,"tag":169,"props":12571,"children":12573},{"className":12572},[177],[12574,12654],{"type":150,"tag":169,"props":12575,"children":12577},{"className":12576},[182],[12578],{"type":150,"tag":185,"props":12579,"children":12580},{"xmlns":187},[12581],{"type":150,"tag":191,"props":12582,"children":12583},{},[12584,12649],{"type":150,"tag":195,"props":12585,"children":12586},{},[12587,12598,12602,12606,12610,12621,12625,12636,12640,12645],{"type":150,"tag":401,"props":12588,"children":12589},{},[12590,12594],{"type":150,"tag":405,"props":12591,"children":12592},{},[12593],{"type":155,"value":759},{"type":150,"tag":405,"props":12595,"children":12596},{},[12597],{"type":155,"value":10046},{"type":150,"tag":417,"props":12599,"children":12600},{},[12601],{"type":155,"value":1045},{"type":150,"tag":405,"props":12603,"children":12604},{},[12605],{"type":155,"value":8100},{"type":150,"tag":417,"props":12607,"children":12608},{"stretchy":207},[12609],{"type":155,"value":972},{"type":150,"tag":401,"props":12611,"children":12612},{},[12613,12617],{"type":150,"tag":405,"props":12614,"children":12615},{},[12616],{"type":155,"value":759},{"type":150,"tag":405,"props":12618,"children":12619},{},[12620],{"type":155,"value":1876},{"type":150,"tag":417,"props":12622,"children":12623},{},[12624],{"type":155,"value":5136},{"type":150,"tag":401,"props":12626,"children":12627},{},[12628,12632],{"type":150,"tag":405,"props":12629,"children":12630},{},[12631],{"type":155,"value":759},{"type":150,"tag":405,"props":12633,"children":12634},{},[12635],{"type":155,"value":1915},{"type":150,"tag":417,"props":12637,"children":12638},{"stretchy":207},[12639],{"type":155,"value":982},{"type":150,"tag":417,"props":12641,"children":12642},{},[12643],{"type":155,"value":12644},"\u003C",{"type":150,"tag":405,"props":12646,"children":12647},{},[12648],{"type":155,"value":759},{"type":150,"tag":223,"props":12650,"children":12651},{"encoding":225},[12652],{"type":155,"value":12653},"d_c = g(d_k + d_v) \u003C d",{"type":150,"tag":169,"props":12655,"children":12657},{"className":12656,"ariaHidden":214},[232],[12658,12736,12824,12907],{"type":150,"tag":169,"props":12659,"children":12661},{"className":12660},[237],[12662,12666,12723,12727,12732],{"type":150,"tag":169,"props":12663,"children":12665},{"className":12664,"style":4935},[242],[],{"type":150,"tag":169,"props":12667,"children":12669},{"className":12668},[248],[12670,12675],{"type":150,"tag":169,"props":12671,"children":12673},{"className":12672},[248,489],[12674],{"type":155,"value":759},{"type":150,"tag":169,"props":12676,"children":12678},{"className":12677},[495],[12679],{"type":150,"tag":169,"props":12680,"children":12682},{"className":12681},[253,254],[12683,12712],{"type":150,"tag":169,"props":12684,"children":12686},{"className":12685},[259],[12687,12707],{"type":150,"tag":169,"props":12688,"children":12690},{"className":12689,"style":5039},[264],[12691],{"type":150,"tag":169,"props":12692,"children":12693},{"style":512},[12694,12698],{"type":150,"tag":169,"props":12695,"children":12697},{"className":12696,"style":517},[274],[],{"type":150,"tag":169,"props":12699,"children":12701},{"className":12700},[522,523,524,525],[12702],{"type":150,"tag":169,"props":12703,"children":12705},{"className":12704},[248,489,525],[12706],{"type":155,"value":10046},{"type":150,"tag":169,"props":12708,"children":12710},{"className":12709},[355],[12711],{"type":155,"value":358},{"type":150,"tag":169,"props":12713,"children":12715},{"className":12714},[259],[12716],{"type":150,"tag":169,"props":12717,"children":12719},{"className":12718,"style":544},[264],[12720],{"type":150,"tag":169,"props":12721,"children":12722},{},[],{"type":150,"tag":169,"props":12724,"children":12726},{"className":12725,"style":839},[558],[],{"type":150,"tag":169,"props":12728,"children":12730},{"className":12729},[844],[12731],{"type":155,"value":1045},{"type":150,"tag":169,"props":12733,"children":12735},{"className":12734,"style":839},[558],[],{"type":150,"tag":169,"props":12737,"children":12739},{"className":12738},[237],[12740,12744,12749,12754,12811,12815,12820],{"type":150,"tag":169,"props":12741,"children":12743},{"className":12742,"style":999},[242],[],{"type":150,"tag":169,"props":12745,"children":12747},{"className":12746,"style":2329},[248,489],[12748],{"type":155,"value":8100},{"type":150,"tag":169,"props":12750,"children":12752},{"className":12751},[1004],[12753],{"type":155,"value":972},{"type":150,"tag":169,"props":12755,"children":12757},{"className":12756},[248],[12758,12763],{"type":150,"tag":169,"props":12759,"children":12761},{"className":12760},[248,489],[12762],{"type":155,"value":759},{"type":150,"tag":169,"props":12764,"children":12766},{"className":12765},[495],[12767],{"type":150,"tag":169,"props":12768,"children":12770},{"className":12769},[253,254],[12771,12800],{"type":150,"tag":169,"props":12772,"children":12774},{"className":12773},[259],[12775,12795],{"type":150,"tag":169,"props":12776,"children":12778},{"className":12777,"style":674},[264],[12779],{"type":150,"tag":169,"props":12780,"children":12781},{"style":512},[12782,12786],{"type":150,"tag":169,"props":12783,"children":12785},{"className":12784,"style":517},[274],[],{"type":150,"tag":169,"props":12787,"children":12789},{"className":12788},[522,523,524,525],[12790],{"type":150,"tag":169,"props":12791,"children":12793},{"className":12792,"style":2427},[248,489,525],[12794],{"type":155,"value":1876},{"type":150,"tag":169,"props":12796,"children":12798},{"className":12797},[355],[12799],{"type":155,"value":358},{"type":150,"tag":169,"props":12801,"children":12803},{"className":12802},[259],[12804],{"type":150,"tag":169,"props":12805,"children":12807},{"className":12806,"style":544},[264],[12808],{"type":150,"tag":169,"props":12809,"children":12810},{},[],{"type":150,"tag":169,"props":12812,"children":12814},{"className":12813,"style":5167},[558],[],{"type":150,"tag":169,"props":12816,"children":12818},{"className":12817},[5172],[12819],{"type":155,"value":5136},{"type":150,"tag":169,"props":12821,"children":12823},{"className":12822,"style":5167},[558],[],{"type":150,"tag":169,"props":12825,"children":12827},{"className":12826},[237],[12828,12832,12889,12894,12898,12903],{"type":150,"tag":169,"props":12829,"children":12831},{"className":12830,"style":999},[242],[],{"type":150,"tag":169,"props":12833,"children":12835},{"className":12834},[248],[12836,12841],{"type":150,"tag":169,"props":12837,"children":12839},{"className":12838},[248,489],[12840],{"type":155,"value":759},{"type":150,"tag":169,"props":12842,"children":12844},{"className":12843},[495],[12845],{"type":150,"tag":169,"props":12846,"children":12848},{"className":12847},[253,254],[12849,12878],{"type":150,"tag":169,"props":12850,"children":12852},{"className":12851},[259],[12853,12873],{"type":150,"tag":169,"props":12854,"children":12856},{"className":12855,"style":5039},[264],[12857],{"type":150,"tag":169,"props":12858,"children":12859},{"style":512},[12860,12864],{"type":150,"tag":169,"props":12861,"children":12863},{"className":12862,"style":517},[274],[],{"type":150,"tag":169,"props":12865,"children":12867},{"className":12866},[522,523,524,525],[12868],{"type":150,"tag":169,"props":12869,"children":12871},{"className":12870,"style":2329},[248,489,525],[12872],{"type":155,"value":1915},{"type":150,"tag":169,"props":12874,"children":12876},{"className":12875},[355],[12877],{"type":155,"value":358},{"type":150,"tag":169,"props":12879,"children":12881},{"className":12880},[259],[12882],{"type":150,"tag":169,"props":12883,"children":12885},{"className":12884,"style":544},[264],[12886],{"type":150,"tag":169,"props":12887,"children":12888},{},[],{"type":150,"tag":169,"props":12890,"children":12892},{"className":12891},[1015],[12893],{"type":155,"value":982},{"type":150,"tag":169,"props":12895,"children":12897},{"className":12896,"style":839},[558],[],{"type":150,"tag":169,"props":12899,"children":12901},{"className":12900},[844],[12902],{"type":155,"value":12644},{"type":150,"tag":169,"props":12904,"children":12906},{"className":12905,"style":839},[558],[],{"type":150,"tag":169,"props":12908,"children":12910},{"className":12909},[237],[12911,12915],{"type":150,"tag":169,"props":12912,"children":12914},{"className":12913,"style":942},[242],[],{"type":150,"tag":169,"props":12916,"children":12918},{"className":12917},[248,489],[12919],{"type":155,"value":759},{"type":155,"value":12921},"，这一步就是在把维度压小。所以 ",{"type":150,"tag":169,"props":12923,"children":12925},{"className":12924},[177],[12926,12971],{"type":150,"tag":169,"props":12927,"children":12929},{"className":12928},[182],[12930],{"type":150,"tag":185,"props":12931,"children":12932},{"xmlns":187},[12933],{"type":150,"tag":191,"props":12934,"children":12935},{},[12936,12966],{"type":150,"tag":195,"props":12937,"children":12938},{},[12939,12950,12955],{"type":150,"tag":401,"props":12940,"children":12941},{},[12942,12946],{"type":150,"tag":405,"props":12943,"children":12944},{},[12945],{"type":155,"value":409},{"type":150,"tag":405,"props":12947,"children":12948},{},[12949],{"type":155,"value":739},{"type":150,"tag":417,"props":12951,"children":12952},{},[12953],{"type":155,"value":12954},"→",{"type":150,"tag":401,"props":12956,"children":12957},{},[12958,12962],{"type":150,"tag":405,"props":12959,"children":12960},{},[12961],{"type":155,"value":10046},{"type":150,"tag":405,"props":12963,"children":12964},{},[12965],{"type":155,"value":739},{"type":150,"tag":223,"props":12967,"children":12968},{"encoding":225},[12969],{"type":155,"value":12970},"x_i \\to c_i",{"type":150,"tag":169,"props":12972,"children":12974},{"className":12973,"ariaHidden":214},[232],[12975,13053],{"type":150,"tag":169,"props":12976,"children":12978},{"className":12977},[237],[12979,12983,13040,13044,13049],{"type":150,"tag":169,"props":12980,"children":12982},{"className":12981,"style":1343},[242],[],{"type":150,"tag":169,"props":12984,"children":12986},{"className":12985},[248],[12987,12992],{"type":150,"tag":169,"props":12988,"children":12990},{"className":12989},[248,489],[12991],{"type":155,"value":409},{"type":150,"tag":169,"props":12993,"children":12995},{"className":12994},[495],[12996],{"type":150,"tag":169,"props":12997,"children":12999},{"className":12998},[253,254],[13000,13029],{"type":150,"tag":169,"props":13001,"children":13003},{"className":13002},[259],[13004,13024],{"type":150,"tag":169,"props":13005,"children":13007},{"className":13006,"style":802},[264],[13008],{"type":150,"tag":169,"props":13009,"children":13010},{"style":512},[13011,13015],{"type":150,"tag":169,"props":13012,"children":13014},{"className":13013,"style":517},[274],[],{"type":150,"tag":169,"props":13016,"children":13018},{"className":13017},[522,523,524,525],[13019],{"type":150,"tag":169,"props":13020,"children":13022},{"className":13021},[248,489,525],[13023],{"type":155,"value":739},{"type":150,"tag":169,"props":13025,"children":13027},{"className":13026},[355],[13028],{"type":155,"value":358},{"type":150,"tag":169,"props":13030,"children":13032},{"className":13031},[259],[13033],{"type":150,"tag":169,"props":13034,"children":13036},{"className":13035,"style":544},[264],[13037],{"type":150,"tag":169,"props":13038,"children":13039},{},[],{"type":150,"tag":169,"props":13041,"children":13043},{"className":13042,"style":839},[558],[],{"type":150,"tag":169,"props":13045,"children":13047},{"className":13046},[844],[13048],{"type":155,"value":12954},{"type":150,"tag":169,"props":13050,"children":13052},{"className":13051,"style":839},[558],[],{"type":150,"tag":169,"props":13054,"children":13056},{"className":13055},[237],[13057,13061],{"type":150,"tag":169,"props":13058,"children":13060},{"className":13059,"style":1343},[242],[],{"type":150,"tag":169,"props":13062,"children":13064},{"className":13063},[248],[13065,13070],{"type":150,"tag":169,"props":13066,"children":13068},{"className":13067},[248,489],[13069],{"type":155,"value":10046},{"type":150,"tag":169,"props":13071,"children":13073},{"className":13072},[495],[13074],{"type":150,"tag":169,"props":13075,"children":13077},{"className":13076},[253,254],[13078,13107],{"type":150,"tag":169,"props":13079,"children":13081},{"className":13080},[259],[13082,13102],{"type":150,"tag":169,"props":13083,"children":13085},{"className":13084,"style":802},[264],[13086],{"type":150,"tag":169,"props":13087,"children":13088},{"style":512},[13089,13093],{"type":150,"tag":169,"props":13090,"children":13092},{"className":13091,"style":517},[274],[],{"type":150,"tag":169,"props":13094,"children":13096},{"className":13095},[522,523,524,525],[13097],{"type":150,"tag":169,"props":13098,"children":13100},{"className":13099},[248,489,525],[13101],{"type":155,"value":739},{"type":150,"tag":169,"props":13103,"children":13105},{"className":13104},[355],[13106],{"type":155,"value":358},{"type":150,"tag":169,"props":13108,"children":13110},{"className":13109},[259],[13111],{"type":150,"tag":169,"props":13112,"children":13114},{"className":13113,"style":544},[264],[13115],{"type":150,"tag":169,"props":13116,"children":13117},{},[],{"type":155,"value":13119}," 本来就是低秩投影。也就是说，光看\"低秩投影\"这一步，GQA 和 MLA 没有区别。MLA 的本质改进不在低秩投影，而在",{"type":150,"tag":158,"props":13121,"children":13122},{},[13123],{"type":155,"value":13124},"低秩投影之后做了什么",{"type":155,"value":13126},"。GQA 投影之后干的事很简单：把向量对半分成 K 和 V，每一份再均分成 ",{"type":150,"tag":169,"props":13128,"children":13130},{"className":13129},[177],[13131,13152],{"type":150,"tag":169,"props":13132,"children":13134},{"className":13133},[182],[13135],{"type":150,"tag":185,"props":13136,"children":13137},{"xmlns":187},[13138],{"type":150,"tag":191,"props":13139,"children":13140},{},[13141,13148],{"type":150,"tag":195,"props":13142,"children":13143},{},[13144],{"type":150,"tag":405,"props":13145,"children":13146},{},[13147],{"type":155,"value":8100},{"type":150,"tag":223,"props":13149,"children":13150},{"encoding":225},[13151],{"type":155,"value":8100},{"type":150,"tag":169,"props":13153,"children":13155},{"className":13154,"ariaHidden":214},[232],[13156],{"type":150,"tag":169,"props":13157,"children":13159},{"className":13158},[237],[13160,13164],{"type":150,"tag":169,"props":13161,"children":13163},{"className":13162,"style":480},[242],[],{"type":150,"tag":169,"props":13165,"children":13167},{"className":13166,"style":2329},[248,489],[13168],{"type":155,"value":8100},{"type":155,"value":13170}," 份，每份复制 ",{"type":150,"tag":169,"props":13172,"children":13174},{"className":13173},[177],[13175,13204],{"type":150,"tag":169,"props":13176,"children":13178},{"className":13177},[182],[13179],{"type":150,"tag":185,"props":13180,"children":13181},{"xmlns":187},[13182],{"type":150,"tag":191,"props":13183,"children":13184},{},[13185,13200],{"type":150,"tag":195,"props":13186,"children":13187},{},[13188,13192,13196],{"type":150,"tag":405,"props":13189,"children":13190},{},[13191],{"type":155,"value":926},{"type":150,"tag":405,"props":13193,"children":13194},{"mathvariant":1796},[13195],{"type":155,"value":4914},{"type":150,"tag":405,"props":13197,"children":13198},{},[13199],{"type":155,"value":8100},{"type":150,"tag":223,"props":13201,"children":13202},{"encoding":225},[13203],{"type":155,"value":8540},{"type":150,"tag":169,"props":13205,"children":13207},{"className":13206,"ariaHidden":214},[232],[13208],{"type":150,"tag":169,"props":13209,"children":13211},{"className":13210},[237],[13212,13216,13221,13226],{"type":150,"tag":169,"props":13213,"children":13215},{"className":13214,"style":999},[242],[],{"type":150,"tag":169,"props":13217,"children":13219},{"className":13218},[248,489],[13220],{"type":155,"value":926},{"type":150,"tag":169,"props":13222,"children":13224},{"className":13223},[248],[13225],{"type":155,"value":4914},{"type":150,"tag":169,"props":13227,"children":13229},{"className":13228,"style":2329},[248,489],[13230],{"type":155,"value":8100},{"type":155,"value":13232}," 次，凑够 ",{"type":150,"tag":169,"props":13234,"children":13236},{"className":13235},[177],[13237,13258],{"type":150,"tag":169,"props":13238,"children":13240},{"className":13239},[182],[13241],{"type":150,"tag":185,"props":13242,"children":13243},{"xmlns":187},[13244],{"type":150,"tag":191,"props":13245,"children":13246},{},[13247,13254],{"type":150,"tag":195,"props":13248,"children":13249},{},[13250],{"type":150,"tag":405,"props":13251,"children":13252},{},[13253],{"type":155,"value":926},{"type":150,"tag":223,"props":13255,"children":13256},{"encoding":225},[13257],{"type":155,"value":926},{"type":150,"tag":169,"props":13259,"children":13261},{"className":13260,"ariaHidden":214},[232],[13262],{"type":150,"tag":169,"props":13263,"children":13265},{"className":13264},[237],[13266,13270],{"type":150,"tag":169,"props":13267,"children":13269},{"className":13268,"style":942},[242],[],{"type":150,"tag":169,"props":13271,"children":13273},{"className":13272},[248,489],[13274],{"type":155,"value":926},{"type":155,"value":13276}," 个 head。分割和复制都是极简的线性变换。",{"type":150,"tag":151,"props":13278,"children":13279},{},[13280],{"type":155,"value":13281},"MLA 的第一步想法就是把它们换成一般的线性变换，让每个 head 拿到的 K、V 各不相同：",{"type":150,"tag":169,"props":13283,"children":13285},{"className":13284},[172],[13286],{"type":150,"tag":169,"props":13287,"children":13289},{"className":13288},[177],[13290,13493],{"type":150,"tag":169,"props":13291,"children":13293},{"className":13292},[182],[13294],{"type":150,"tag":185,"props":13295,"children":13296},{"xmlns":187,"display":188},[13297],{"type":150,"tag":191,"props":13298,"children":13299},{},[13300,13488],{"type":150,"tag":195,"props":13301,"children":13302},{},[13303,13329,13333,13344,13370,13374,13392,13396,13399,13425,13429,13440,13466,13470],{"type":150,"tag":1224,"props":13304,"children":13305},{},[13306,13310,13314],{"type":150,"tag":405,"props":13307,"children":13308},{},[13309],{"type":155,"value":1876},{"type":150,"tag":405,"props":13311,"children":13312},{},[13313],{"type":155,"value":739},{"type":150,"tag":195,"props":13315,"children":13316},{},[13317,13321,13325],{"type":150,"tag":417,"props":13318,"children":13319},{"stretchy":207},[13320],{"type":155,"value":972},{"type":150,"tag":405,"props":13322,"children":13323},{},[13324],{"type":155,"value":977},{"type":150,"tag":417,"props":13326,"children":13327},{"stretchy":207},[13328],{"type":155,"value":982},{"type":150,"tag":417,"props":13330,"children":13331},{},[13332],{"type":155,"value":1045},{"type":150,"tag":401,"props":13334,"children":13335},{},[13336,13340],{"type":150,"tag":405,"props":13337,"children":13338},{},[13339],{"type":155,"value":10046},{"type":150,"tag":405,"props":13341,"children":13342},{},[13343],{"type":155,"value":739},{"type":150,"tag":1224,"props":13345,"children":13346},{},[13347,13351,13355],{"type":150,"tag":405,"props":13348,"children":13349},{},[13350],{"type":155,"value":3443},{"type":150,"tag":405,"props":13352,"children":13353},{},[13354],{"type":155,"value":1876},{"type":150,"tag":195,"props":13356,"children":13357},{},[13358,13362,13366],{"type":150,"tag":417,"props":13359,"children":13360},{"stretchy":207},[13361],{"type":155,"value":972},{"type":150,"tag":405,"props":13363,"children":13364},{},[13365],{"type":155,"value":977},{"type":150,"tag":417,"props":13367,"children":13368},{"stretchy":207},[13369],{"type":155,"value":982},{"type":150,"tag":417,"props":13371,"children":13372},{},[13373],{"type":155,"value":744},{"type":150,"tag":746,"props":13375,"children":13376},{},[13377,13381],{"type":150,"tag":405,"props":13378,"children":13379},{"mathvariant":751},[13380],{"type":155,"value":754},{"type":150,"tag":401,"props":13382,"children":13383},{},[13384,13388],{"type":150,"tag":405,"props":13385,"children":13386},{},[13387],{"type":155,"value":759},{"type":150,"tag":405,"props":13389,"children":13390},{},[13391],{"type":155,"value":1876},{"type":150,"tag":417,"props":13393,"children":13394},{"separator":214},[13395],{"type":155,"value":421},{"type":150,"tag":558,"props":13397,"children":13398},{"width":3491},[],{"type":150,"tag":1224,"props":13400,"children":13401},{},[13402,13406,13410],{"type":150,"tag":405,"props":13403,"children":13404},{},[13405],{"type":155,"value":1915},{"type":150,"tag":405,"props":13407,"children":13408},{},[13409],{"type":155,"value":739},{"type":150,"tag":195,"props":13411,"children":13412},{},[13413,13417,13421],{"type":150,"tag":417,"props":13414,"children":13415},{"stretchy":207},[13416],{"type":155,"value":972},{"type":150,"tag":405,"props":13418,"children":13419},{},[13420],{"type":155,"value":977},{"type":150,"tag":417,"props":13422,"children":13423},{"stretchy":207},[13424],{"type":155,"value":982},{"type":150,"tag":417,"props":13426,"children":13427},{},[13428],{"type":155,"value":1045},{"type":150,"tag":401,"props":13430,"children":13431},{},[13432,13436],{"type":150,"tag":405,"props":13433,"children":13434},{},[13435],{"type":155,"value":10046},{"type":150,"tag":405,"props":13437,"children":13438},{},[13439],{"type":155,"value":739},{"type":150,"tag":1224,"props":13441,"children":13442},{},[13443,13447,13451],{"type":150,"tag":405,"props":13444,"children":13445},{},[13446],{"type":155,"value":3443},{"type":150,"tag":405,"props":13448,"children":13449},{},[13450],{"type":155,"value":1915},{"type":150,"tag":195,"props":13452,"children":13453},{},[13454,13458,13462],{"type":150,"tag":417,"props":13455,"children":13456},{"stretchy":207},[13457],{"type":155,"value":972},{"type":150,"tag":405,"props":13459,"children":13460},{},[13461],{"type":155,"value":977},{"type":150,"tag":417,"props":13463,"children":13464},{"stretchy":207},[13465],{"type":155,"value":982},{"type":150,"tag":417,"props":13467,"children":13468},{},[13469],{"type":155,"value":744},{"type":150,"tag":746,"props":13471,"children":13472},{},[13473,13477],{"type":150,"tag":405,"props":13474,"children":13475},{"mathvariant":751},[13476],{"type":155,"value":754},{"type":150,"tag":401,"props":13478,"children":13479},{},[13480,13484],{"type":150,"tag":405,"props":13481,"children":13482},{},[13483],{"type":155,"value":759},{"type":150,"tag":405,"props":13485,"children":13486},{},[13487],{"type":155,"value":1915},{"type":150,"tag":223,"props":13489,"children":13490},{"encoding":225},[13491],{"type":155,"value":13492},"k_i^{(s)} = c_i W_k^{(s)} \\in \\mathbb{R}^{d_k}, \\quad v_i^{(s)} = c_i W_v^{(s)} \\in \\mathbb{R}^{d_v}",{"type":150,"tag":169,"props":13494,"children":13496},{"className":13495,"ariaHidden":214},[232],[13497,13605,13770,13988,14153],{"type":150,"tag":169,"props":13498,"children":13500},{"className":13499},[237],[13501,13505,13592,13596,13601],{"type":150,"tag":169,"props":13502,"children":13504},{"className":13503,"style":3694},[242],[],{"type":150,"tag":169,"props":13506,"children":13508},{"className":13507},[248],[13509,13514],{"type":150,"tag":169,"props":13510,"children":13512},{"className":13511,"style":2427},[248,489],[13513],{"type":155,"value":1876},{"type":150,"tag":169,"props":13515,"children":13517},{"className":13516},[495],[13518],{"type":150,"tag":169,"props":13519,"children":13521},{"className":13520},[253,254],[13522,13581],{"type":150,"tag":169,"props":13523,"children":13525},{"className":13524},[259],[13526,13576],{"type":150,"tag":169,"props":13527,"children":13529},{"className":13528,"style":1454},[264],[13530,13546],{"type":150,"tag":169,"props":13531,"children":13532},{"style":2898},[13533,13537],{"type":150,"tag":169,"props":13534,"children":13536},{"className":13535,"style":517},[274],[],{"type":150,"tag":169,"props":13538,"children":13540},{"className":13539},[522,523,524,525],[13541],{"type":150,"tag":169,"props":13542,"children":13544},{"className":13543},[248,489,525],[13545],{"type":155,"value":739},{"type":150,"tag":169,"props":13547,"children":13548},{"style":1475},[13549,13553],{"type":150,"tag":169,"props":13550,"children":13552},{"className":13551,"style":517},[274],[],{"type":150,"tag":169,"props":13554,"children":13556},{"className":13555},[522,523,524,525],[13557],{"type":150,"tag":169,"props":13558,"children":13560},{"className":13559},[248,525],[13561,13566,13571],{"type":150,"tag":169,"props":13562,"children":13564},{"className":13563},[1004,525],[13565],{"type":155,"value":972},{"type":150,"tag":169,"props":13567,"children":13569},{"className":13568},[248,489,525],[13570],{"type":155,"value":977},{"type":150,"tag":169,"props":13572,"children":13574},{"className":13573},[1015,525],[13575],{"type":155,"value":982},{"type":150,"tag":169,"props":13577,"children":13579},{"className":13578},[355],[13580],{"type":155,"value":358},{"type":150,"tag":169,"props":13582,"children":13584},{"className":13583},[259],[13585],{"type":150,"tag":169,"props":13586,"children":13588},{"className":13587,"style":2960},[264],[13589],{"type":150,"tag":169,"props":13590,"children":13591},{},[],{"type":150,"tag":169,"props":13593,"children":13595},{"className":13594,"style":839},[558],[],{"type":150,"tag":169,"props":13597,"children":13599},{"className":13598},[844],[13600],{"type":155,"value":1045},{"type":150,"tag":169,"props":13602,"children":13604},{"className":13603,"style":839},[558],[],{"type":150,"tag":169,"props":13606,"children":13608},{"className":13607},[237],[13609,13613,13670,13757,13761,13766],{"type":150,"tag":169,"props":13610,"children":13612},{"className":13611,"style":4200},[242],[],{"type":150,"tag":169,"props":13614,"children":13616},{"className":13615},[248],[13617,13622],{"type":150,"tag":169,"props":13618,"children":13620},{"className":13619},[248,489],[13621],{"type":155,"value":10046},{"type":150,"tag":169,"props":13623,"children":13625},{"className":13624},[495],[13626],{"type":150,"tag":169,"props":13627,"children":13629},{"className":13628},[253,254],[13630,13659],{"type":150,"tag":169,"props":13631,"children":13633},{"className":13632},[259],[13634,13654],{"type":150,"tag":169,"props":13635,"children":13637},{"className":13636,"style":802},[264],[13638],{"type":150,"tag":169,"props":13639,"children":13640},{"style":512},[13641,13645],{"type":150,"tag":169,"props":13642,"children":13644},{"className":13643,"style":517},[274],[],{"type":150,"tag":169,"props":13646,"children":13648},{"className":13647},[522,523,524,525],[13649],{"type":150,"tag":169,"props":13650,"children":13652},{"className":13651},[248,489,525],[13653],{"type":155,"value":739},{"type":150,"tag":169,"props":13655,"children":13657},{"className":13656},[355],[13658],{"type":155,"value":358},{"type":150,"tag":169,"props":13660,"children":13662},{"className":13661},[259],[13663],{"type":150,"tag":169,"props":13664,"children":13666},{"className":13665,"style":544},[264],[13667],{"type":150,"tag":169,"props":13668,"children":13669},{},[],{"type":150,"tag":169,"props":13671,"children":13673},{"className":13672},[248],[13674,13679],{"type":150,"tag":169,"props":13675,"children":13677},{"className":13676,"style":3869},[248,489],[13678],{"type":155,"value":3443},{"type":150,"tag":169,"props":13680,"children":13682},{"className":13681},[495],[13683],{"type":150,"tag":169,"props":13684,"children":13686},{"className":13685},[253,254],[13687,13746],{"type":150,"tag":169,"props":13688,"children":13690},{"className":13689},[259],[13691,13741],{"type":150,"tag":169,"props":13692,"children":13694},{"className":13693,"style":1454},[264],[13695,13711],{"type":150,"tag":169,"props":13696,"children":13697},{"style":4286},[13698,13702],{"type":150,"tag":169,"props":13699,"children":13701},{"className":13700,"style":517},[274],[],{"type":150,"tag":169,"props":13703,"children":13705},{"className":13704},[522,523,524,525],[13706],{"type":150,"tag":169,"props":13707,"children":13709},{"className":13708,"style":2427},[248,489,525],[13710],{"type":155,"value":1876},{"type":150,"tag":169,"props":13712,"children":13713},{"style":1475},[13714,13718],{"type":150,"tag":169,"props":13715,"children":13717},{"className":13716,"style":517},[274],[],{"type":150,"tag":169,"props":13719,"children":13721},{"className":13720},[522,523,524,525],[13722],{"type":150,"tag":169,"props":13723,"children":13725},{"className":13724},[248,525],[13726,13731,13736],{"type":150,"tag":169,"props":13727,"children":13729},{"className":13728},[1004,525],[13730],{"type":155,"value":972},{"type":150,"tag":169,"props":13732,"children":13734},{"className":13733},[248,489,525],[13735],{"type":155,"value":977},{"type":150,"tag":169,"props":13737,"children":13739},{"className":13738},[1015,525],[13740],{"type":155,"value":982},{"type":150,"tag":169,"props":13742,"children":13744},{"className":13743},[355],[13745],{"type":155,"value":358},{"type":150,"tag":169,"props":13747,"children":13749},{"className":13748},[259],[13750],{"type":150,"tag":169,"props":13751,"children":13753},{"className":13752,"style":4343},[264],[13754],{"type":150,"tag":169,"props":13755,"children":13756},{},[],{"type":150,"tag":169,"props":13758,"children":13760},{"className":13759,"style":839},[558],[],{"type":150,"tag":169,"props":13762,"children":13764},{"className":13763},[844],[13765],{"type":155,"value":744},{"type":150,"tag":169,"props":13767,"children":13769},{"className":13768,"style":839},[558],[],{"type":150,"tag":169,"props":13771,"children":13773},{"className":13772},[237],[13774,13778,13875,13880,13884,13888,13975,13979,13984],{"type":150,"tag":169,"props":13775,"children":13777},{"className":13776,"style":3694},[242],[],{"type":150,"tag":169,"props":13779,"children":13781},{"className":13780},[248],[13782,13787],{"type":150,"tag":169,"props":13783,"children":13785},{"className":13784},[248,867],[13786],{"type":155,"value":754},{"type":150,"tag":169,"props":13788,"children":13790},{"className":13789},[495],[13791],{"type":150,"tag":169,"props":13792,"children":13794},{"className":13793},[253],[13795],{"type":150,"tag":169,"props":13796,"children":13798},{"className":13797},[259],[13799],{"type":150,"tag":169,"props":13800,"children":13802},{"className":13801,"style":3999},[264],[13803],{"type":150,"tag":169,"props":13804,"children":13805},{"style":3908},[13806,13810],{"type":150,"tag":169,"props":13807,"children":13809},{"className":13808,"style":517},[274],[],{"type":150,"tag":169,"props":13811,"children":13813},{"className":13812},[522,523,524,525],[13814],{"type":150,"tag":169,"props":13815,"children":13817},{"className":13816},[248,525],[13818],{"type":150,"tag":169,"props":13819,"children":13821},{"className":13820},[248,525],[13822,13827],{"type":150,"tag":169,"props":13823,"children":13825},{"className":13824},[248,489,525],[13826],{"type":155,"value":759},{"type":150,"tag":169,"props":13828,"children":13830},{"className":13829},[495],[13831],{"type":150,"tag":169,"props":13832,"children":13834},{"className":13833},[253,254],[13835,13864],{"type":150,"tag":169,"props":13836,"children":13838},{"className":13837},[259],[13839,13859],{"type":150,"tag":169,"props":13840,"children":13842},{"className":13841,"style":4040},[264],[13843],{"type":150,"tag":169,"props":13844,"children":13845},{"style":4044},[13846,13850],{"type":150,"tag":169,"props":13847,"children":13849},{"className":13848,"style":4049},[274],[],{"type":150,"tag":169,"props":13851,"children":13853},{"className":13852},[522,4054,4055,525],[13854],{"type":150,"tag":169,"props":13855,"children":13857},{"className":13856,"style":2427},[248,489,525],[13858],{"type":155,"value":1876},{"type":150,"tag":169,"props":13860,"children":13862},{"className":13861},[355],[13863],{"type":155,"value":358},{"type":150,"tag":169,"props":13865,"children":13867},{"className":13866},[259],[13868],{"type":150,"tag":169,"props":13869,"children":13871},{"className":13870,"style":4074},[264],[13872],{"type":150,"tag":169,"props":13873,"children":13874},{},[],{"type":150,"tag":169,"props":13876,"children":13878},{"className":13877},[552],[13879],{"type":155,"value":421},{"type":150,"tag":169,"props":13881,"children":13883},{"className":13882,"style":4087},[558],[],{"type":150,"tag":169,"props":13885,"children":13887},{"className":13886,"style":559},[558],[],{"type":150,"tag":169,"props":13889,"children":13891},{"className":13890},[248],[13892,13897],{"type":150,"tag":169,"props":13893,"children":13895},{"className":13894,"style":2329},[248,489],[13896],{"type":155,"value":1915},{"type":150,"tag":169,"props":13898,"children":13900},{"className":13899},[495],[13901],{"type":150,"tag":169,"props":13902,"children":13904},{"className":13903},[253,254],[13905,13964],{"type":150,"tag":169,"props":13906,"children":13908},{"className":13907},[259],[13909,13959],{"type":150,"tag":169,"props":13910,"children":13912},{"className":13911,"style":1454},[264],[13913,13929],{"type":150,"tag":169,"props":13914,"children":13915},{"style":3292},[13916,13920],{"type":150,"tag":169,"props":13917,"children":13919},{"className":13918,"style":517},[274],[],{"type":150,"tag":169,"props":13921,"children":13923},{"className":13922},[522,523,524,525],[13924],{"type":150,"tag":169,"props":13925,"children":13927},{"className":13926},[248,489,525],[13928],{"type":155,"value":739},{"type":150,"tag":169,"props":13930,"children":13931},{"style":1475},[13932,13936],{"type":150,"tag":169,"props":13933,"children":13935},{"className":13934,"style":517},[274],[],{"type":150,"tag":169,"props":13937,"children":13939},{"className":13938},[522,523,524,525],[13940],{"type":150,"tag":169,"props":13941,"children":13943},{"className":13942},[248,525],[13944,13949,13954],{"type":150,"tag":169,"props":13945,"children":13947},{"className":13946},[1004,525],[13948],{"type":155,"value":972},{"type":150,"tag":169,"props":13950,"children":13952},{"className":13951},[248,489,525],[13953],{"type":155,"value":977},{"type":150,"tag":169,"props":13955,"children":13957},{"className":13956},[1015,525],[13958],{"type":155,"value":982},{"type":150,"tag":169,"props":13960,"children":13962},{"className":13961},[355],[13963],{"type":155,"value":358},{"type":150,"tag":169,"props":13965,"children":13967},{"className":13966},[259],[13968],{"type":150,"tag":169,"props":13969,"children":13971},{"className":13970,"style":2960},[264],[13972],{"type":150,"tag":169,"props":13973,"children":13974},{},[],{"type":150,"tag":169,"props":13976,"children":13978},{"className":13977,"style":839},[558],[],{"type":150,"tag":169,"props":13980,"children":13982},{"className":13981},[844],[13983],{"type":155,"value":1045},{"type":150,"tag":169,"props":13985,"children":13987},{"className":13986,"style":839},[558],[],{"type":150,"tag":169,"props":13989,"children":13991},{"className":13990},[237],[13992,13996,14053,14140,14144,14149],{"type":150,"tag":169,"props":13993,"children":13995},{"className":13994,"style":4586},[242],[],{"type":150,"tag":169,"props":13997,"children":13999},{"className":13998},[248],[14000,14005],{"type":150,"tag":169,"props":14001,"children":14003},{"className":14002},[248,489],[14004],{"type":155,"value":10046},{"type":150,"tag":169,"props":14006,"children":14008},{"className":14007},[495],[14009],{"type":150,"tag":169,"props":14010,"children":14012},{"className":14011},[253,254],[14013,14042],{"type":150,"tag":169,"props":14014,"children":14016},{"className":14015},[259],[14017,14037],{"type":150,"tag":169,"props":14018,"children":14020},{"className":14019,"style":802},[264],[14021],{"type":150,"tag":169,"props":14022,"children":14023},{"style":512},[14024,14028],{"type":150,"tag":169,"props":14025,"children":14027},{"className":14026,"style":517},[274],[],{"type":150,"tag":169,"props":14029,"children":14031},{"className":14030},[522,523,524,525],[14032],{"type":150,"tag":169,"props":14033,"children":14035},{"className":14034},[248,489,525],[14036],{"type":155,"value":739},{"type":150,"tag":169,"props":14038,"children":14040},{"className":14039},[355],[14041],{"type":155,"value":358},{"type":150,"tag":169,"props":14043,"children":14045},{"className":14044},[259],[14046],{"type":150,"tag":169,"props":14047,"children":14049},{"className":14048,"style":544},[264],[14050],{"type":150,"tag":169,"props":14051,"children":14052},{},[],{"type":150,"tag":169,"props":14054,"children":14056},{"className":14055},[248],[14057,14062],{"type":150,"tag":169,"props":14058,"children":14060},{"className":14059,"style":3869},[248,489],[14061],{"type":155,"value":3443},{"type":150,"tag":169,"props":14063,"children":14065},{"className":14064},[495],[14066],{"type":150,"tag":169,"props":14067,"children":14069},{"className":14068},[253,254],[14070,14129],{"type":150,"tag":169,"props":14071,"children":14073},{"className":14072},[259],[14074,14124],{"type":150,"tag":169,"props":14075,"children":14077},{"className":14076,"style":3887},[264],[14078,14094],{"type":150,"tag":169,"props":14079,"children":14080},{"style":3891},[14081,14085],{"type":150,"tag":169,"props":14082,"children":14084},{"className":14083,"style":517},[274],[],{"type":150,"tag":169,"props":14086,"children":14088},{"className":14087},[522,523,524,525],[14089],{"type":150,"tag":169,"props":14090,"children":14092},{"className":14091,"style":2329},[248,489,525],[14093],{"type":155,"value":1915},{"type":150,"tag":169,"props":14095,"children":14096},{"style":3908},[14097,14101],{"type":150,"tag":169,"props":14098,"children":14100},{"className":14099,"style":517},[274],[],{"type":150,"tag":169,"props":14102,"children":14104},{"className":14103},[522,523,524,525],[14105],{"type":150,"tag":169,"props":14106,"children":14108},{"className":14107},[248,525],[14109,14114,14119],{"type":150,"tag":169,"props":14110,"children":14112},{"className":14111},[1004,525],[14113],{"type":155,"value":972},{"type":150,"tag":169,"props":14115,"children":14117},{"className":14116},[248,489,525],[14118],{"type":155,"value":977},{"type":150,"tag":169,"props":14120,"children":14122},{"className":14121},[1015,525],[14123],{"type":155,"value":982},{"type":150,"tag":169,"props":14125,"children":14127},{"className":14126},[355],[14128],{"type":155,"value":358},{"type":150,"tag":169,"props":14130,"children":14132},{"className":14131},[259],[14133],{"type":150,"tag":169,"props":14134,"children":14136},{"className":14135,"style":4728},[264],[14137],{"type":150,"tag":169,"props":14138,"children":14139},{},[],{"type":150,"tag":169,"props":14141,"children":14143},{"className":14142,"style":839},[558],[],{"type":150,"tag":169,"props":14145,"children":14147},{"className":14146},[844],[14148],{"type":155,"value":744},{"type":150,"tag":169,"props":14150,"children":14152},{"className":14151,"style":839},[558],[],{"type":150,"tag":169,"props":14154,"children":14156},{"className":14155},[237],[14157,14161],{"type":150,"tag":169,"props":14158,"children":14160},{"className":14159,"style":3999},[242],[],{"type":150,"tag":169,"props":14162,"children":14164},{"className":14163},[248],[14165,14170],{"type":150,"tag":169,"props":14166,"children":14168},{"className":14167},[248,867],[14169],{"type":155,"value":754},{"type":150,"tag":169,"props":14171,"children":14173},{"className":14172},[495],[14174],{"type":150,"tag":169,"props":14175,"children":14177},{"className":14176},[253],[14178],{"type":150,"tag":169,"props":14179,"children":14181},{"className":14180},[259],[14182],{"type":150,"tag":169,"props":14183,"children":14185},{"className":14184,"style":3999},[264],[14186],{"type":150,"tag":169,"props":14187,"children":14188},{"style":3908},[14189,14193],{"type":150,"tag":169,"props":14190,"children":14192},{"className":14191,"style":517},[274],[],{"type":150,"tag":169,"props":14194,"children":14196},{"className":14195},[522,523,524,525],[14197],{"type":150,"tag":169,"props":14198,"children":14200},{"className":14199},[248,525],[14201],{"type":150,"tag":169,"props":14202,"children":14204},{"className":14203},[248,525],[14205,14210],{"type":150,"tag":169,"props":14206,"children":14208},{"className":14207},[248,489,525],[14209],{"type":155,"value":759},{"type":150,"tag":169,"props":14211,"children":14213},{"className":14212},[495],[14214],{"type":150,"tag":169,"props":14215,"children":14217},{"className":14216},[253,254],[14218,14247],{"type":150,"tag":169,"props":14219,"children":14221},{"className":14220},[259],[14222,14242],{"type":150,"tag":169,"props":14223,"children":14225},{"className":14224,"style":4818},[264],[14226],{"type":150,"tag":169,"props":14227,"children":14228},{"style":4822},[14229,14233],{"type":150,"tag":169,"props":14230,"children":14232},{"className":14231,"style":4049},[274],[],{"type":150,"tag":169,"props":14234,"children":14236},{"className":14235},[522,4054,4055,525],[14237],{"type":150,"tag":169,"props":14238,"children":14240},{"className":14239,"style":2329},[248,489,525],[14241],{"type":155,"value":1915},{"type":150,"tag":169,"props":14243,"children":14245},{"className":14244},[355],[14246],{"type":155,"value":358},{"type":150,"tag":169,"props":14248,"children":14250},{"className":14249},[259],[14251],{"type":150,"tag":169,"props":14252,"children":14254},{"className":14253,"style":4849},[264],[14255],{"type":150,"tag":169,"props":14256,"children":14257},{},[],{"type":150,"tag":151,"props":14259,"children":14260},{},[14261,14263,14448],{"type":155,"value":14262},"但这样问题就来了。我们缓存的是投影后的 ",{"type":150,"tag":169,"props":14264,"children":14266},{"className":14265},[177],[14267,14311],{"type":150,"tag":169,"props":14268,"children":14270},{"className":14269},[182],[14271],{"type":150,"tag":185,"props":14272,"children":14273},{"xmlns":187},[14274],{"type":150,"tag":191,"props":14275,"children":14276},{},[14277,14306],{"type":150,"tag":195,"props":14278,"children":14279},{},[14280,14291,14295],{"type":150,"tag":401,"props":14281,"children":14282},{},[14283,14287],{"type":150,"tag":405,"props":14284,"children":14285},{},[14286],{"type":155,"value":1876},{"type":150,"tag":405,"props":14288,"children":14289},{},[14290],{"type":155,"value":739},{"type":150,"tag":417,"props":14292,"children":14293},{"separator":214},[14294],{"type":155,"value":421},{"type":150,"tag":401,"props":14296,"children":14297},{},[14298,14302],{"type":150,"tag":405,"props":14299,"children":14300},{},[14301],{"type":155,"value":1915},{"type":150,"tag":405,"props":14303,"children":14304},{},[14305],{"type":155,"value":739},{"type":150,"tag":223,"props":14307,"children":14308},{"encoding":225},[14309],{"type":155,"value":14310},"k_i, v_i",{"type":150,"tag":169,"props":14312,"children":14314},{"className":14313,"ariaHidden":214},[232],[14315],{"type":150,"tag":169,"props":14316,"children":14318},{"className":14317},[237],[14319,14323,14381,14386,14390],{"type":150,"tag":169,"props":14320,"children":14322},{"className":14321,"style":1121},[242],[],{"type":150,"tag":169,"props":14324,"children":14326},{"className":14325},[248],[14327,14332],{"type":150,"tag":169,"props":14328,"children":14330},{"className":14329,"style":2427},[248,489],[14331],{"type":155,"value":1876},{"type":150,"tag":169,"props":14333,"children":14335},{"className":14334},[495],[14336],{"type":150,"tag":169,"props":14337,"children":14339},{"className":14338},[253,254],[14340,14370],{"type":150,"tag":169,"props":14341,"children":14343},{"className":14342},[259],[14344,14365],{"type":150,"tag":169,"props":14345,"children":14347},{"className":14346,"style":802},[264],[14348],{"type":150,"tag":169,"props":14349,"children":14351},{"style":14350},"top:-2.55em;margin-left:-0.0315em;margin-right:0.05em;",[14352,14356],{"type":150,"tag":169,"props":14353,"children":14355},{"className":14354,"style":517},[274],[],{"type":150,"tag":169,"props":14357,"children":14359},{"className":14358},[522,523,524,525],[14360],{"type":150,"tag":169,"props":14361,"children":14363},{"className":14362},[248,489,525],[14364],{"type":155,"value":739},{"type":150,"tag":169,"props":14366,"children":14368},{"className":14367},[355],[14369],{"type":155,"value":358},{"type":150,"tag":169,"props":14371,"children":14373},{"className":14372},[259],[14374],{"type":150,"tag":169,"props":14375,"children":14377},{"className":14376,"style":544},[264],[14378],{"type":150,"tag":169,"props":14379,"children":14380},{},[],{"type":150,"tag":169,"props":14382,"children":14384},{"className":14383},[552],[14385],{"type":155,"value":421},{"type":150,"tag":169,"props":14387,"children":14389},{"className":14388,"style":559},[558],[],{"type":150,"tag":169,"props":14391,"children":14393},{"className":14392},[248],[14394,14399],{"type":150,"tag":169,"props":14395,"children":14397},{"className":14396,"style":2329},[248,489],[14398],{"type":155,"value":1915},{"type":150,"tag":169,"props":14400,"children":14402},{"className":14401},[495],[14403],{"type":150,"tag":169,"props":14404,"children":14406},{"className":14405},[253,254],[14407,14437],{"type":150,"tag":169,"props":14408,"children":14410},{"className":14409},[259],[14411,14432],{"type":150,"tag":169,"props":14412,"children":14414},{"className":14413,"style":802},[264],[14415],{"type":150,"tag":169,"props":14416,"children":14418},{"style":14417},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[14419,14423],{"type":150,"tag":169,"props":14420,"children":14422},{"className":14421,"style":517},[274],[],{"type":150,"tag":169,"props":14424,"children":14426},{"className":14425},[522,523,524,525],[14427],{"type":150,"tag":169,"props":14428,"children":14430},{"className":14429},[248,489,525],[14431],{"type":155,"value":739},{"type":150,"tag":169,"props":14433,"children":14435},{"className":14434},[355],[14436],{"type":155,"value":358},{"type":150,"tag":169,"props":14438,"children":14440},{"className":14439},[259],[14441],{"type":150,"tag":169,"props":14442,"children":14444},{"className":14443,"style":544},[264],[14445],{"type":150,"tag":169,"props":14446,"children":14447},{},[],{"type":155,"value":14449},"，一旦它们各不相同，KV Cache 就恢复成 MHA 那么大，GQA 的初衷全丢了。MLA 的做法是找一个恒等变换绕过去。在推理阶段：",{"type":150,"tag":169,"props":14451,"children":14453},{"className":14452},[172],[14454],{"type":150,"tag":169,"props":14455,"children":14457},{"className":14456},[177],[14458,14727],{"type":150,"tag":169,"props":14459,"children":14461},{"className":14460},[182],[14462],{"type":150,"tag":185,"props":14463,"children":14464},{"xmlns":187,"display":188},[14465],{"type":150,"tag":191,"props":14466,"children":14467},{},[14468,14722],{"type":150,"tag":195,"props":14469,"children":14470},{},[14471,14497,14527,14531,14535,14546,14572,14576,14580,14591,14617,14628,14632,14643,14647,14673,14703,14707],{"type":150,"tag":1224,"props":14472,"children":14473},{},[14474,14478,14482],{"type":150,"tag":405,"props":14475,"children":14476},{},[14477],{"type":155,"value":1845},{"type":150,"tag":405,"props":14479,"children":14480},{},[14481],{"type":155,"value":1213},{"type":150,"tag":195,"props":14483,"children":14484},{},[14485,14489,14493],{"type":150,"tag":417,"props":14486,"children":14487},{"stretchy":207},[14488],{"type":155,"value":972},{"type":150,"tag":405,"props":14490,"children":14491},{},[14492],{"type":155,"value":977},{"type":150,"tag":417,"props":14494,"children":14495},{"stretchy":207},[14496],{"type":155,"value":982},{"type":150,"tag":1224,"props":14498,"children":14499},{},[14500,14504,14508],{"type":150,"tag":405,"props":14501,"children":14502},{},[14503],{"type":155,"value":1876},{"type":150,"tag":405,"props":14505,"children":14506},{},[14507],{"type":155,"value":739},{"type":150,"tag":195,"props":14509,"children":14510},{},[14511,14515,14519,14523],{"type":150,"tag":417,"props":14512,"children":14513},{"stretchy":207},[14514],{"type":155,"value":972},{"type":150,"tag":405,"props":14516,"children":14517},{},[14518],{"type":155,"value":977},{"type":150,"tag":417,"props":14520,"children":14521},{"stretchy":207},[14522],{"type":155,"value":982},{"type":150,"tag":405,"props":14524,"children":14525},{"mathvariant":1796},[14526],{"type":155,"value":2052},{"type":150,"tag":417,"props":14528,"children":14529},{},[14530],{"type":155,"value":1045},{"type":150,"tag":417,"props":14532,"children":14533},{"stretchy":207},[14534],{"type":155,"value":972},{"type":150,"tag":401,"props":14536,"children":14537},{},[14538,14542],{"type":150,"tag":405,"props":14539,"children":14540},{},[14541],{"type":155,"value":409},{"type":150,"tag":405,"props":14543,"children":14544},{},[14545],{"type":155,"value":1213},{"type":150,"tag":1224,"props":14547,"children":14548},{},[14549,14553,14557],{"type":150,"tag":405,"props":14550,"children":14551},{},[14552],{"type":155,"value":3443},{"type":150,"tag":405,"props":14554,"children":14555},{},[14556],{"type":155,"value":1845},{"type":150,"tag":195,"props":14558,"children":14559},{},[14560,14564,14568],{"type":150,"tag":417,"props":14561,"children":14562},{"stretchy":207},[14563],{"type":155,"value":972},{"type":150,"tag":405,"props":14565,"children":14566},{},[14567],{"type":155,"value":977},{"type":150,"tag":417,"props":14569,"children":14570},{"stretchy":207},[14571],{"type":155,"value":982},{"type":150,"tag":417,"props":14573,"children":14574},{"stretchy":207},[14575],{"type":155,"value":982},{"type":150,"tag":417,"props":14577,"children":14578},{"stretchy":207},[14579],{"type":155,"value":972},{"type":150,"tag":401,"props":14581,"children":14582},{},[14583,14587],{"type":150,"tag":405,"props":14584,"children":14585},{},[14586],{"type":155,"value":10046},{"type":150,"tag":405,"props":14588,"children":14589},{},[14590],{"type":155,"value":739},{"type":150,"tag":1224,"props":14592,"children":14593},{},[14594,14598,14602],{"type":150,"tag":405,"props":14595,"children":14596},{},[14597],{"type":155,"value":3443},{"type":150,"tag":405,"props":14599,"children":14600},{},[14601],{"type":155,"value":1876},{"type":150,"tag":195,"props":14603,"children":14604},{},[14605,14609,14613],{"type":150,"tag":417,"props":14606,"children":14607},{"stretchy":207},[14608],{"type":155,"value":972},{"type":150,"tag":405,"props":14610,"children":14611},{},[14612],{"type":155,"value":977},{"type":150,"tag":417,"props":14614,"children":14615},{"stretchy":207},[14616],{"type":155,"value":982},{"type":150,"tag":746,"props":14618,"children":14619},{},[14620,14624],{"type":150,"tag":417,"props":14621,"children":14622},{"stretchy":207},[14623],{"type":155,"value":982},{"type":150,"tag":405,"props":14625,"children":14626},{"mathvariant":1796},[14627],{"type":155,"value":2052},{"type":150,"tag":417,"props":14629,"children":14630},{},[14631],{"type":155,"value":1045},{"type":150,"tag":401,"props":14633,"children":14634},{},[14635,14639],{"type":150,"tag":405,"props":14636,"children":14637},{},[14638],{"type":155,"value":409},{"type":150,"tag":405,"props":14640,"children":14641},{},[14642],{"type":155,"value":1213},{"type":150,"tag":417,"props":14644,"children":14645},{"stretchy":207},[14646],{"type":155,"value":972},{"type":150,"tag":1224,"props":14648,"children":14649},{},[14650,14654,14658],{"type":150,"tag":405,"props":14651,"children":14652},{},[14653],{"type":155,"value":3443},{"type":150,"tag":405,"props":14655,"children":14656},{},[14657],{"type":155,"value":1845},{"type":150,"tag":195,"props":14659,"children":14660},{},[14661,14665,14669],{"type":150,"tag":417,"props":14662,"children":14663},{"stretchy":207},[14664],{"type":155,"value":972},{"type":150,"tag":405,"props":14666,"children":14667},{},[14668],{"type":155,"value":977},{"type":150,"tag":417,"props":14670,"children":14671},{"stretchy":207},[14672],{"type":155,"value":982},{"type":150,"tag":1224,"props":14674,"children":14675},{},[14676,14680,14684],{"type":150,"tag":405,"props":14677,"children":14678},{},[14679],{"type":155,"value":3443},{"type":150,"tag":405,"props":14681,"children":14682},{},[14683],{"type":155,"value":1876},{"type":150,"tag":195,"props":14685,"children":14686},{},[14687,14691,14695,14699],{"type":150,"tag":417,"props":14688,"children":14689},{"stretchy":207},[14690],{"type":155,"value":972},{"type":150,"tag":405,"props":14692,"children":14693},{},[14694],{"type":155,"value":977},{"type":150,"tag":417,"props":14696,"children":14697},{"stretchy":207},[14698],{"type":155,"value":982},{"type":150,"tag":405,"props":14700,"children":14701},{"mathvariant":1796},[14702],{"type":155,"value":2052},{"type":150,"tag":417,"props":14704,"children":14705},{"stretchy":207},[14706],{"type":155,"value":982},{"type":150,"tag":1224,"props":14708,"children":14709},{},[14710,14714,14718],{"type":150,"tag":405,"props":14711,"children":14712},{},[14713],{"type":155,"value":10046},{"type":150,"tag":405,"props":14715,"children":14716},{},[14717],{"type":155,"value":739},{"type":150,"tag":405,"props":14719,"children":14720},{"mathvariant":1796},[14721],{"type":155,"value":2052},{"type":150,"tag":223,"props":14723,"children":14724},{"encoding":225},[14725],{"type":155,"value":14726},"q_t^{(s)} k_i^{(s)\\top} = (x_t W_q^{(s)})(c_i W_k^{(s)})^\\top = x_t (W_q^{(s)} W_k^{(s)\\top}) c_i^\\top",{"type":150,"tag":169,"props":14728,"children":14730},{"className":14729,"ariaHidden":214},[232],[14731,14931,15297],{"type":150,"tag":169,"props":14732,"children":14734},{"className":14733},[237],[14735,14739,14826,14918,14922,14927],{"type":150,"tag":169,"props":14736,"children":14738},{"className":14737,"style":3694},[242],[],{"type":150,"tag":169,"props":14740,"children":14742},{"className":14741},[248],[14743,14748],{"type":150,"tag":169,"props":14744,"children":14746},{"className":14745,"style":2329},[248,489],[14747],{"type":155,"value":1845},{"type":150,"tag":169,"props":14749,"children":14751},{"className":14750},[495],[14752],{"type":150,"tag":169,"props":14753,"children":14755},{"className":14754},[253,254],[14756,14815],{"type":150,"tag":169,"props":14757,"children":14759},{"className":14758},[259],[14760,14810],{"type":150,"tag":169,"props":14761,"children":14763},{"className":14762,"style":1454},[264],[14764,14780],{"type":150,"tag":169,"props":14765,"children":14766},{"style":2350},[14767,14771],{"type":150,"tag":169,"props":14768,"children":14770},{"className":14769,"style":517},[274],[],{"type":150,"tag":169,"props":14772,"children":14774},{"className":14773},[522,523,524,525],[14775],{"type":150,"tag":169,"props":14776,"children":14778},{"className":14777},[248,489,525],[14779],{"type":155,"value":1213},{"type":150,"tag":169,"props":14781,"children":14782},{"style":1475},[14783,14787],{"type":150,"tag":169,"props":14784,"children":14786},{"className":14785,"style":517},[274],[],{"type":150,"tag":169,"props":14788,"children":14790},{"className":14789},[522,523,524,525],[14791],{"type":150,"tag":169,"props":14792,"children":14794},{"className":14793},[248,525],[14795,14800,14805],{"type":150,"tag":169,"props":14796,"children":14798},{"className":14797},[1004,525],[14799],{"type":155,"value":972},{"type":150,"tag":169,"props":14801,"children":14803},{"className":14802},[248,489,525],[14804],{"type":155,"value":977},{"type":150,"tag":169,"props":14806,"children":14808},{"className":14807},[1015,525],[14809],{"type":155,"value":982},{"type":150,"tag":169,"props":14811,"children":14813},{"className":14812},[355],[14814],{"type":155,"value":358},{"type":150,"tag":169,"props":14816,"children":14818},{"className":14817},[259],[14819],{"type":150,"tag":169,"props":14820,"children":14822},{"className":14821,"style":1516},[264],[14823],{"type":150,"tag":169,"props":14824,"children":14825},{},[],{"type":150,"tag":169,"props":14827,"children":14829},{"className":14828},[248],[14830,14835],{"type":150,"tag":169,"props":14831,"children":14833},{"className":14832,"style":2427},[248,489],[14834],{"type":155,"value":1876},{"type":150,"tag":169,"props":14836,"children":14838},{"className":14837},[495],[14839],{"type":150,"tag":169,"props":14840,"children":14842},{"className":14841},[253,254],[14843,14907],{"type":150,"tag":169,"props":14844,"children":14846},{"className":14845},[259],[14847,14902],{"type":150,"tag":169,"props":14848,"children":14850},{"className":14849,"style":1454},[264],[14851,14867],{"type":150,"tag":169,"props":14852,"children":14853},{"style":2898},[14854,14858],{"type":150,"tag":169,"props":14855,"children":14857},{"className":14856,"style":517},[274],[],{"type":150,"tag":169,"props":14859,"children":14861},{"className":14860},[522,523,524,525],[14862],{"type":150,"tag":169,"props":14863,"children":14865},{"className":14864},[248,489,525],[14866],{"type":155,"value":739},{"type":150,"tag":169,"props":14868,"children":14869},{"style":1475},[14870,14874],{"type":150,"tag":169,"props":14871,"children":14873},{"className":14872,"style":517},[274],[],{"type":150,"tag":169,"props":14875,"children":14877},{"className":14876},[522,523,524,525],[14878],{"type":150,"tag":169,"props":14879,"children":14881},{"className":14880},[248,525],[14882,14887,14892,14897],{"type":150,"tag":169,"props":14883,"children":14885},{"className":14884},[1004,525],[14886],{"type":155,"value":972},{"type":150,"tag":169,"props":14888,"children":14890},{"className":14889},[248,489,525],[14891],{"type":155,"value":977},{"type":150,"tag":169,"props":14893,"children":14895},{"className":14894},[1015,525],[14896],{"type":155,"value":982},{"type":150,"tag":169,"props":14898,"children":14900},{"className":14899},[248,525],[14901],{"type":155,"value":2052},{"type":150,"tag":169,"props":14903,"children":14905},{"className":14904},[355],[14906],{"type":155,"value":358},{"type":150,"tag":169,"props":14908,"children":14910},{"className":14909},[259],[14911],{"type":150,"tag":169,"props":14912,"children":14914},{"className":14913,"style":2960},[264],[14915],{"type":150,"tag":169,"props":14916,"children":14917},{},[],{"type":150,"tag":169,"props":14919,"children":14921},{"className":14920,"style":839},[558],[],{"type":150,"tag":169,"props":14923,"children":14925},{"className":14924},[844],[14926],{"type":155,"value":1045},{"type":150,"tag":169,"props":14928,"children":14930},{"className":14929,"style":839},[558],[],{"type":150,"tag":169,"props":14932,"children":14934},{"className":14933},[237],[14935,14940,14945,15002,15089,15094,15099,15156,15243,15284,15288,15293],{"type":150,"tag":169,"props":14936,"children":14939},{"className":14937,"style":14938},[242],"height:1.4279em;vertical-align:-0.3831em;",[],{"type":150,"tag":169,"props":14941,"children":14943},{"className":14942},[1004],[14944],{"type":155,"value":972},{"type":150,"tag":169,"props":14946,"children":14948},{"className":14947},[248],[14949,14954],{"type":150,"tag":169,"props":14950,"children":14952},{"className":14951},[248,489],[14953],{"type":155,"value":409},{"type":150,"tag":169,"props":14955,"children":14957},{"className":14956},[495],[14958],{"type":150,"tag":169,"props":14959,"children":14961},{"className":14960},[253,254],[14962,14991],{"type":150,"tag":169,"props":14963,"children":14965},{"className":14964},[259],[14966,14986],{"type":150,"tag":169,"props":14967,"children":14969},{"className":14968,"style":1369},[264],[14970],{"type":150,"tag":169,"props":14971,"children":14972},{"style":512},[14973,14977],{"type":150,"tag":169,"props":14974,"children":14976},{"className":14975,"style":517},[274],[],{"type":150,"tag":169,"props":14978,"children":14980},{"className":14979},[522,523,524,525],[14981],{"type":150,"tag":169,"props":14982,"children":14984},{"className":14983},[248,489,525],[14985],{"type":155,"value":1213},{"type":150,"tag":169,"props":14987,"children":14989},{"className":14988},[355],[14990],{"type":155,"value":358},{"type":150,"tag":169,"props":14992,"children":14994},{"className":14993},[259],[14995],{"type":150,"tag":169,"props":14996,"children":14998},{"className":14997,"style":544},[264],[14999],{"type":150,"tag":169,"props":15000,"children":15001},{},[],{"type":150,"tag":169,"props":15003,"children":15005},{"className":15004},[248],[15006,15011],{"type":150,"tag":169,"props":15007,"children":15009},{"className":15008,"style":3869},[248,489],[15010],{"type":155,"value":3443},{"type":150,"tag":169,"props":15012,"children":15014},{"className":15013},[495],[15015],{"type":150,"tag":169,"props":15016,"children":15018},{"className":15017},[253,254],[15019,15078],{"type":150,"tag":169,"props":15020,"children":15022},{"className":15021},[259],[15023,15073],{"type":150,"tag":169,"props":15024,"children":15026},{"className":15025,"style":3887},[264],[15027,15043],{"type":150,"tag":169,"props":15028,"children":15029},{"style":3891},[15030,15034],{"type":150,"tag":169,"props":15031,"children":15033},{"className":15032,"style":517},[274],[],{"type":150,"tag":169,"props":15035,"children":15037},{"className":15036},[522,523,524,525],[15038],{"type":150,"tag":169,"props":15039,"children":15041},{"className":15040,"style":2329},[248,489,525],[15042],{"type":155,"value":1845},{"type":150,"tag":169,"props":15044,"children":15045},{"style":3908},[15046,15050],{"type":150,"tag":169,"props":15047,"children":15049},{"className":15048,"style":517},[274],[],{"type":150,"tag":169,"props":15051,"children":15053},{"className":15052},[522,523,524,525],[15054],{"type":150,"tag":169,"props":15055,"children":15057},{"className":15056},[248,525],[15058,15063,15068],{"type":150,"tag":169,"props":15059,"children":15061},{"className":15060},[1004,525],[15062],{"type":155,"value":972},{"type":150,"tag":169,"props":15064,"children":15066},{"className":15065},[248,489,525],[15067],{"type":155,"value":977},{"type":150,"tag":169,"props":15069,"children":15071},{"className":15070},[1015,525],[15072],{"type":155,"value":982},{"type":150,"tag":169,"props":15074,"children":15076},{"className":15075},[355],[15077],{"type":155,"value":358},{"type":150,"tag":169,"props":15079,"children":15081},{"className":15080},[259],[15082],{"type":150,"tag":169,"props":15083,"children":15085},{"className":15084,"style":3949},[264],[15086],{"type":150,"tag":169,"props":15087,"children":15088},{},[],{"type":150,"tag":169,"props":15090,"children":15092},{"className":15091},[1015],[15093],{"type":155,"value":982},{"type":150,"tag":169,"props":15095,"children":15097},{"className":15096},[1004],[15098],{"type":155,"value":972},{"type":150,"tag":169,"props":15100,"children":15102},{"className":15101},[248],[15103,15108],{"type":150,"tag":169,"props":15104,"children":15106},{"className":15105},[248,489],[15107],{"type":155,"value":10046},{"type":150,"tag":169,"props":15109,"children":15111},{"className":15110},[495],[15112],{"type":150,"tag":169,"props":15113,"children":15115},{"className":15114},[253,254],[15116,15145],{"type":150,"tag":169,"props":15117,"children":15119},{"className":15118},[259],[15120,15140],{"type":150,"tag":169,"props":15121,"children":15123},{"className":15122,"style":802},[264],[15124],{"type":150,"tag":169,"props":15125,"children":15126},{"style":512},[15127,15131],{"type":150,"tag":169,"props":15128,"children":15130},{"className":15129,"style":517},[274],[],{"type":150,"tag":169,"props":15132,"children":15134},{"className":15133},[522,523,524,525],[15135],{"type":150,"tag":169,"props":15136,"children":15138},{"className":15137},[248,489,525],[15139],{"type":155,"value":739},{"type":150,"tag":169,"props":15141,"children":15143},{"className":15142},[355],[15144],{"type":155,"value":358},{"type":150,"tag":169,"props":15146,"children":15148},{"className":15147},[259],[15149],{"type":150,"tag":169,"props":15150,"children":15152},{"className":15151,"style":544},[264],[15153],{"type":150,"tag":169,"props":15154,"children":15155},{},[],{"type":150,"tag":169,"props":15157,"children":15159},{"className":15158},[248],[15160,15165],{"type":150,"tag":169,"props":15161,"children":15163},{"className":15162,"style":3869},[248,489],[15164],{"type":155,"value":3443},{"type":150,"tag":169,"props":15166,"children":15168},{"className":15167},[495],[15169],{"type":150,"tag":169,"props":15170,"children":15172},{"className":15171},[253,254],[15173,15232],{"type":150,"tag":169,"props":15174,"children":15176},{"className":15175},[259],[15177,15227],{"type":150,"tag":169,"props":15178,"children":15180},{"className":15179,"style":1454},[264],[15181,15197],{"type":150,"tag":169,"props":15182,"children":15183},{"style":4286},[15184,15188],{"type":150,"tag":169,"props":15185,"children":15187},{"className":15186,"style":517},[274],[],{"type":150,"tag":169,"props":15189,"children":15191},{"className":15190},[522,523,524,525],[15192],{"type":150,"tag":169,"props":15193,"children":15195},{"className":15194,"style":2427},[248,489,525],[15196],{"type":155,"value":1876},{"type":150,"tag":169,"props":15198,"children":15199},{"style":1475},[15200,15204],{"type":150,"tag":169,"props":15201,"children":15203},{"className":15202,"style":517},[274],[],{"type":150,"tag":169,"props":15205,"children":15207},{"className":15206},[522,523,524,525],[15208],{"type":150,"tag":169,"props":15209,"children":15211},{"className":15210},[248,525],[15212,15217,15222],{"type":150,"tag":169,"props":15213,"children":15215},{"className":15214},[1004,525],[15216],{"type":155,"value":972},{"type":150,"tag":169,"props":15218,"children":15220},{"className":15219},[248,489,525],[15221],{"type":155,"value":977},{"type":150,"tag":169,"props":15223,"children":15225},{"className":15224},[1015,525],[15226],{"type":155,"value":982},{"type":150,"tag":169,"props":15228,"children":15230},{"className":15229},[355],[15231],{"type":155,"value":358},{"type":150,"tag":169,"props":15233,"children":15235},{"className":15234},[259],[15236],{"type":150,"tag":169,"props":15237,"children":15239},{"className":15238,"style":4343},[264],[15240],{"type":150,"tag":169,"props":15241,"children":15242},{},[],{"type":150,"tag":169,"props":15244,"children":15246},{"className":15245},[1015],[15247,15252],{"type":150,"tag":169,"props":15248,"children":15250},{"className":15249},[1015],[15251],{"type":155,"value":982},{"type":150,"tag":169,"props":15253,"children":15255},{"className":15254},[495],[15256],{"type":150,"tag":169,"props":15257,"children":15259},{"className":15258},[253],[15260],{"type":150,"tag":169,"props":15261,"children":15263},{"className":15262},[259],[15264],{"type":150,"tag":169,"props":15265,"children":15267},{"className":15266,"style":3999},[264],[15268],{"type":150,"tag":169,"props":15269,"children":15270},{"style":3908},[15271,15275],{"type":150,"tag":169,"props":15272,"children":15274},{"className":15273,"style":517},[274],[],{"type":150,"tag":169,"props":15276,"children":15278},{"className":15277},[522,523,524,525],[15279],{"type":150,"tag":169,"props":15280,"children":15282},{"className":15281},[248,525],[15283],{"type":155,"value":2052},{"type":150,"tag":169,"props":15285,"children":15287},{"className":15286,"style":839},[558],[],{"type":150,"tag":169,"props":15289,"children":15291},{"className":15290},[844],[15292],{"type":155,"value":1045},{"type":150,"tag":169,"props":15294,"children":15296},{"className":15295,"style":839},[558],[],{"type":150,"tag":169,"props":15298,"children":15300},{"className":15299},[237],[15301,15305,15362,15367,15454,15546,15551],{"type":150,"tag":169,"props":15302,"children":15304},{"className":15303,"style":14938},[242],[],{"type":150,"tag":169,"props":15306,"children":15308},{"className":15307},[248],[15309,15314],{"type":150,"tag":169,"props":15310,"children":15312},{"className":15311},[248,489],[15313],{"type":155,"value":409},{"type":150,"tag":169,"props":15315,"children":15317},{"className":15316},[495],[15318],{"type":150,"tag":169,"props":15319,"children":15321},{"className":15320},[253,254],[15322,15351],{"type":150,"tag":169,"props":15323,"children":15325},{"className":15324},[259],[15326,15346],{"type":150,"tag":169,"props":15327,"children":15329},{"className":15328,"style":1369},[264],[15330],{"type":150,"tag":169,"props":15331,"children":15332},{"style":512},[15333,15337],{"type":150,"tag":169,"props":15334,"children":15336},{"className":15335,"style":517},[274],[],{"type":150,"tag":169,"props":15338,"children":15340},{"className":15339},[522,523,524,525],[15341],{"type":150,"tag":169,"props":15342,"children":15344},{"className":15343},[248,489,525],[15345],{"type":155,"value":1213},{"type":150,"tag":169,"props":15347,"children":15349},{"className":15348},[355],[15350],{"type":155,"value":358},{"type":150,"tag":169,"props":15352,"children":15354},{"className":15353},[259],[15355],{"type":150,"tag":169,"props":15356,"children":15358},{"className":15357,"style":544},[264],[15359],{"type":150,"tag":169,"props":15360,"children":15361},{},[],{"type":150,"tag":169,"props":15363,"children":15365},{"className":15364},[1004],[15366],{"type":155,"value":972},{"type":150,"tag":169,"props":15368,"children":15370},{"className":15369},[248],[15371,15376],{"type":150,"tag":169,"props":15372,"children":15374},{"className":15373,"style":3869},[248,489],[15375],{"type":155,"value":3443},{"type":150,"tag":169,"props":15377,"children":15379},{"className":15378},[495],[15380],{"type":150,"tag":169,"props":15381,"children":15383},{"className":15382},[253,254],[15384,15443],{"type":150,"tag":169,"props":15385,"children":15387},{"className":15386},[259],[15388,15438],{"type":150,"tag":169,"props":15389,"children":15391},{"className":15390,"style":3887},[264],[15392,15408],{"type":150,"tag":169,"props":15393,"children":15394},{"style":3891},[15395,15399],{"type":150,"tag":169,"props":15396,"children":15398},{"className":15397,"style":517},[274],[],{"type":150,"tag":169,"props":15400,"children":15402},{"className":15401},[522,523,524,525],[15403],{"type":150,"tag":169,"props":15404,"children":15406},{"className":15405,"style":2329},[248,489,525],[15407],{"type":155,"value":1845},{"type":150,"tag":169,"props":15409,"children":15410},{"style":3908},[15411,15415],{"type":150,"tag":169,"props":15412,"children":15414},{"className":15413,"style":517},[274],[],{"type":150,"tag":169,"props":15416,"children":15418},{"className":15417},[522,523,524,525],[15419],{"type":150,"tag":169,"props":15420,"children":15422},{"className":15421},[248,525],[15423,15428,15433],{"type":150,"tag":169,"props":15424,"children":15426},{"className":15425},[1004,525],[15427],{"type":155,"value":972},{"type":150,"tag":169,"props":15429,"children":15431},{"className":15430},[248,489,525],[15432],{"type":155,"value":977},{"type":150,"tag":169,"props":15434,"children":15436},{"className":15435},[1015,525],[15437],{"type":155,"value":982},{"type":150,"tag":169,"props":15439,"children":15441},{"className":15440},[355],[15442],{"type":155,"value":358},{"type":150,"tag":169,"props":15444,"children":15446},{"className":15445},[259],[15447],{"type":150,"tag":169,"props":15448,"children":15450},{"className":15449,"style":3949},[264],[15451],{"type":150,"tag":169,"props":15452,"children":15453},{},[],{"type":150,"tag":169,"props":15455,"children":15457},{"className":15456},[248],[15458,15463],{"type":150,"tag":169,"props":15459,"children":15461},{"className":15460,"style":3869},[248,489],[15462],{"type":155,"value":3443},{"type":150,"tag":169,"props":15464,"children":15466},{"className":15465},[495],[15467],{"type":150,"tag":169,"props":15468,"children":15470},{"className":15469},[253,254],[15471,15535],{"type":150,"tag":169,"props":15472,"children":15474},{"className":15473},[259],[15475,15530],{"type":150,"tag":169,"props":15476,"children":15478},{"className":15477,"style":1454},[264],[15479,15495],{"type":150,"tag":169,"props":15480,"children":15481},{"style":4286},[15482,15486],{"type":150,"tag":169,"props":15483,"children":15485},{"className":15484,"style":517},[274],[],{"type":150,"tag":169,"props":15487,"children":15489},{"className":15488},[522,523,524,525],[15490],{"type":150,"tag":169,"props":15491,"children":15493},{"className":15492,"style":2427},[248,489,525],[15494],{"type":155,"value":1876},{"type":150,"tag":169,"props":15496,"children":15497},{"style":1475},[15498,15502],{"type":150,"tag":169,"props":15499,"children":15501},{"className":15500,"style":517},[274],[],{"type":150,"tag":169,"props":15503,"children":15505},{"className":15504},[522,523,524,525],[15506],{"type":150,"tag":169,"props":15507,"children":15509},{"className":15508},[248,525],[15510,15515,15520,15525],{"type":150,"tag":169,"props":15511,"children":15513},{"className":15512},[1004,525],[15514],{"type":155,"value":972},{"type":150,"tag":169,"props":15516,"children":15518},{"className":15517},[248,489,525],[15519],{"type":155,"value":977},{"type":150,"tag":169,"props":15521,"children":15523},{"className":15522},[1015,525],[15524],{"type":155,"value":982},{"type":150,"tag":169,"props":15526,"children":15528},{"className":15527},[248,525],[15529],{"type":155,"value":2052},{"type":150,"tag":169,"props":15531,"children":15533},{"className":15532},[355],[15534],{"type":155,"value":358},{"type":150,"tag":169,"props":15536,"children":15538},{"className":15537},[259],[15539],{"type":150,"tag":169,"props":15540,"children":15542},{"className":15541,"style":4343},[264],[15543],{"type":150,"tag":169,"props":15544,"children":15545},{},[],{"type":150,"tag":169,"props":15547,"children":15549},{"className":15548},[1015],[15550],{"type":155,"value":982},{"type":150,"tag":169,"props":15552,"children":15554},{"className":15553},[248],[15555,15560],{"type":150,"tag":169,"props":15556,"children":15558},{"className":15557},[248,489],[15559],{"type":155,"value":10046},{"type":150,"tag":169,"props":15561,"children":15563},{"className":15562},[495],[15564],{"type":150,"tag":169,"props":15565,"children":15567},{"className":15566},[253,254],[15568,15614],{"type":150,"tag":169,"props":15569,"children":15571},{"className":15570},[259],[15572,15609],{"type":150,"tag":169,"props":15573,"children":15575},{"className":15574,"style":3999},[264],[15576,15593],{"type":150,"tag":169,"props":15577,"children":15579},{"style":15578},"top:-2.453em;margin-left:0em;margin-right:0.05em;",[15580,15584],{"type":150,"tag":169,"props":15581,"children":15583},{"className":15582,"style":517},[274],[],{"type":150,"tag":169,"props":15585,"children":15587},{"className":15586},[522,523,524,525],[15588],{"type":150,"tag":169,"props":15589,"children":15591},{"className":15590},[248,489,525],[15592],{"type":155,"value":739},{"type":150,"tag":169,"props":15594,"children":15595},{"style":3908},[15596,15600],{"type":150,"tag":169,"props":15597,"children":15599},{"className":15598,"style":517},[274],[],{"type":150,"tag":169,"props":15601,"children":15603},{"className":15602},[522,523,524,525],[15604],{"type":150,"tag":169,"props":15605,"children":15607},{"className":15606},[248,525],[15608],{"type":155,"value":2052},{"type":150,"tag":169,"props":15610,"children":15612},{"className":15611},[355],[15613],{"type":155,"value":358},{"type":150,"tag":169,"props":15615,"children":15617},{"className":15616},[259],[15618],{"type":150,"tag":169,"props":15619,"children":15621},{"className":15620,"style":4728},[264],[15622],{"type":150,"tag":169,"props":15623,"children":15624},{},[],{"type":150,"tag":151,"props":15626,"children":15627},{},[15628,15630,15901,15903,16004,16006,16108,16110,16465,16467,16615,16617,16719,16721,16822,16824,16925,16927,17028,17030,17090],{"type":155,"value":15629},"这意味着可以把 ",{"type":150,"tag":169,"props":15631,"children":15633},{"className":15632},[177],[15634,15708],{"type":150,"tag":169,"props":15635,"children":15637},{"className":15636},[182],[15638],{"type":150,"tag":185,"props":15639,"children":15640},{"xmlns":187},[15641],{"type":150,"tag":191,"props":15642,"children":15643},{},[15644,15703],{"type":150,"tag":195,"props":15645,"children":15646},{},[15647,15673],{"type":150,"tag":1224,"props":15648,"children":15649},{},[15650,15654,15658],{"type":150,"tag":405,"props":15651,"children":15652},{},[15653],{"type":155,"value":3443},{"type":150,"tag":405,"props":15655,"children":15656},{},[15657],{"type":155,"value":1845},{"type":150,"tag":195,"props":15659,"children":15660},{},[15661,15665,15669],{"type":150,"tag":417,"props":15662,"children":15663},{"stretchy":207},[15664],{"type":155,"value":972},{"type":150,"tag":405,"props":15666,"children":15667},{},[15668],{"type":155,"value":977},{"type":150,"tag":417,"props":15670,"children":15671},{"stretchy":207},[15672],{"type":155,"value":982},{"type":150,"tag":1224,"props":15674,"children":15675},{},[15676,15680,15684],{"type":150,"tag":405,"props":15677,"children":15678},{},[15679],{"type":155,"value":3443},{"type":150,"tag":405,"props":15681,"children":15682},{},[15683],{"type":155,"value":1876},{"type":150,"tag":195,"props":15685,"children":15686},{},[15687,15691,15695,15699],{"type":150,"tag":417,"props":15688,"children":15689},{"stretchy":207},[15690],{"type":155,"value":972},{"type":150,"tag":405,"props":15692,"children":15693},{},[15694],{"type":155,"value":977},{"type":150,"tag":417,"props":15696,"children":15697},{"stretchy":207},[15698],{"type":155,"value":982},{"type":150,"tag":405,"props":15700,"children":15701},{"mathvariant":1796},[15702],{"type":155,"value":2052},{"type":150,"tag":223,"props":15704,"children":15705},{"encoding":225},[15706],{"type":155,"value":15707},"W_q^{(s)} W_k^{(s)\\top}",{"type":150,"tag":169,"props":15709,"children":15711},{"className":15710,"ariaHidden":214},[232],[15712],{"type":150,"tag":169,"props":15713,"children":15715},{"className":15714},[237],[15716,15720,15809],{"type":150,"tag":169,"props":15717,"children":15719},{"className":15718,"style":4200},[242],[],{"type":150,"tag":169,"props":15721,"children":15723},{"className":15722},[248],[15724,15729],{"type":150,"tag":169,"props":15725,"children":15727},{"className":15726,"style":3869},[248,489],[15728],{"type":155,"value":3443},{"type":150,"tag":169,"props":15730,"children":15732},{"className":15731},[495],[15733],{"type":150,"tag":169,"props":15734,"children":15736},{"className":15735},[253,254],[15737,15797],{"type":150,"tag":169,"props":15738,"children":15740},{"className":15739},[259],[15741,15792],{"type":150,"tag":169,"props":15742,"children":15744},{"className":15743,"style":1454},[264],[15745,15762],{"type":150,"tag":169,"props":15746,"children":15748},{"style":15747},"top:-2.5834em;margin-left:-0.1389em;margin-right:0.05em;",[15749,15753],{"type":150,"tag":169,"props":15750,"children":15752},{"className":15751,"style":517},[274],[],{"type":150,"tag":169,"props":15754,"children":15756},{"className":15755},[522,523,524,525],[15757],{"type":150,"tag":169,"props":15758,"children":15760},{"className":15759,"style":2329},[248,489,525],[15761],{"type":155,"value":1845},{"type":150,"tag":169,"props":15763,"children":15764},{"style":1475},[15765,15769],{"type":150,"tag":169,"props":15766,"children":15768},{"className":15767,"style":517},[274],[],{"type":150,"tag":169,"props":15770,"children":15772},{"className":15771},[522,523,524,525],[15773],{"type":150,"tag":169,"props":15774,"children":15776},{"className":15775},[248,525],[15777,15782,15787],{"type":150,"tag":169,"props":15778,"children":15780},{"className":15779},[1004,525],[15781],{"type":155,"value":972},{"type":150,"tag":169,"props":15783,"children":15785},{"className":15784},[248,489,525],[15786],{"type":155,"value":977},{"type":150,"tag":169,"props":15788,"children":15790},{"className":15789},[1015,525],[15791],{"type":155,"value":982},{"type":150,"tag":169,"props":15793,"children":15795},{"className":15794},[355],[15796],{"type":155,"value":358},{"type":150,"tag":169,"props":15798,"children":15800},{"className":15799},[259],[15801],{"type":150,"tag":169,"props":15802,"children":15805},{"className":15803,"style":15804},[264],"height:0.2527em;",[15806],{"type":150,"tag":169,"props":15807,"children":15808},{},[],{"type":150,"tag":169,"props":15810,"children":15812},{"className":15811},[248],[15813,15818],{"type":150,"tag":169,"props":15814,"children":15816},{"className":15815,"style":3869},[248,489],[15817],{"type":155,"value":3443},{"type":150,"tag":169,"props":15819,"children":15821},{"className":15820},[495],[15822],{"type":150,"tag":169,"props":15823,"children":15825},{"className":15824},[253,254],[15826,15890],{"type":150,"tag":169,"props":15827,"children":15829},{"className":15828},[259],[15830,15885],{"type":150,"tag":169,"props":15831,"children":15833},{"className":15832,"style":1454},[264],[15834,15850],{"type":150,"tag":169,"props":15835,"children":15836},{"style":4286},[15837,15841],{"type":150,"tag":169,"props":15838,"children":15840},{"className":15839,"style":517},[274],[],{"type":150,"tag":169,"props":15842,"children":15844},{"className":15843},[522,523,524,525],[15845],{"type":150,"tag":169,"props":15846,"children":15848},{"className":15847,"style":2427},[248,489,525],[15849],{"type":155,"value":1876},{"type":150,"tag":169,"props":15851,"children":15852},{"style":1475},[15853,15857],{"type":150,"tag":169,"props":15854,"children":15856},{"className":15855,"style":517},[274],[],{"type":150,"tag":169,"props":15858,"children":15860},{"className":15859},[522,523,524,525],[15861],{"type":150,"tag":169,"props":15862,"children":15864},{"className":15863},[248,525],[15865,15870,15875,15880],{"type":150,"tag":169,"props":15866,"children":15868},{"className":15867},[1004,525],[15869],{"type":155,"value":972},{"type":150,"tag":169,"props":15871,"children":15873},{"className":15872},[248,489,525],[15874],{"type":155,"value":977},{"type":150,"tag":169,"props":15876,"children":15878},{"className":15877},[1015,525],[15879],{"type":155,"value":982},{"type":150,"tag":169,"props":15881,"children":15883},{"className":15882},[248,525],[15884],{"type":155,"value":2052},{"type":150,"tag":169,"props":15886,"children":15888},{"className":15887},[355],[15889],{"type":155,"value":358},{"type":150,"tag":169,"props":15891,"children":15893},{"className":15892},[259],[15894],{"type":150,"tag":169,"props":15895,"children":15897},{"className":15896,"style":4343},[264],[15898],{"type":150,"tag":169,"props":15899,"children":15900},{},[],{"type":155,"value":15902}," 合并起来当作 Q 的投影矩阵，那么 ",{"type":150,"tag":169,"props":15904,"children":15906},{"className":15905},[177],[15907,15935],{"type":150,"tag":169,"props":15908,"children":15910},{"className":15909},[182],[15911],{"type":150,"tag":185,"props":15912,"children":15913},{"xmlns":187},[15914],{"type":150,"tag":191,"props":15915,"children":15916},{},[15917,15931],{"type":150,"tag":195,"props":15918,"children":15919},{},[15920],{"type":150,"tag":401,"props":15921,"children":15922},{},[15923,15927],{"type":150,"tag":405,"props":15924,"children":15925},{},[15926],{"type":155,"value":10046},{"type":150,"tag":405,"props":15928,"children":15929},{},[15930],{"type":155,"value":739},{"type":150,"tag":223,"props":15932,"children":15933},{"encoding":225},[15934],{"type":155,"value":10055},{"type":150,"tag":169,"props":15936,"children":15938},{"className":15937,"ariaHidden":214},[232],[15939],{"type":150,"tag":169,"props":15940,"children":15942},{"className":15941},[237],[15943,15947],{"type":150,"tag":169,"props":15944,"children":15946},{"className":15945,"style":1343},[242],[],{"type":150,"tag":169,"props":15948,"children":15950},{"className":15949},[248],[15951,15956],{"type":150,"tag":169,"props":15952,"children":15954},{"className":15953},[248,489],[15955],{"type":155,"value":10046},{"type":150,"tag":169,"props":15957,"children":15959},{"className":15958},[495],[15960],{"type":150,"tag":169,"props":15961,"children":15963},{"className":15962},[253,254],[15964,15993],{"type":150,"tag":169,"props":15965,"children":15967},{"className":15966},[259],[15968,15988],{"type":150,"tag":169,"props":15969,"children":15971},{"className":15970,"style":802},[264],[15972],{"type":150,"tag":169,"props":15973,"children":15974},{"style":512},[15975,15979],{"type":150,"tag":169,"props":15976,"children":15978},{"className":15977,"style":517},[274],[],{"type":150,"tag":169,"props":15980,"children":15982},{"className":15981},[522,523,524,525],[15983],{"type":150,"tag":169,"props":15984,"children":15986},{"className":15985},[248,489,525],[15987],{"type":155,"value":739},{"type":150,"tag":169,"props":15989,"children":15991},{"className":15990},[355],[15992],{"type":155,"value":358},{"type":150,"tag":169,"props":15994,"children":15996},{"className":15995},[259],[15997],{"type":150,"tag":169,"props":15998,"children":16000},{"className":15999,"style":544},[264],[16001],{"type":150,"tag":169,"props":16002,"children":16003},{},[],{"type":155,"value":16005}," 就取代了 ",{"type":150,"tag":169,"props":16007,"children":16009},{"className":16008},[177],[16010,16039],{"type":150,"tag":169,"props":16011,"children":16013},{"className":16012},[182],[16014],{"type":150,"tag":185,"props":16015,"children":16016},{"xmlns":187},[16017],{"type":150,"tag":191,"props":16018,"children":16019},{},[16020,16034],{"type":150,"tag":195,"props":16021,"children":16022},{},[16023],{"type":150,"tag":401,"props":16024,"children":16025},{},[16026,16030],{"type":150,"tag":405,"props":16027,"children":16028},{},[16029],{"type":155,"value":1876},{"type":150,"tag":405,"props":16031,"children":16032},{},[16033],{"type":155,"value":739},{"type":150,"tag":223,"props":16035,"children":16036},{"encoding":225},[16037],{"type":155,"value":16038},"k_i",{"type":150,"tag":169,"props":16040,"children":16042},{"className":16041,"ariaHidden":214},[232],[16043],{"type":150,"tag":169,"props":16044,"children":16046},{"className":16045},[237],[16047,16051],{"type":150,"tag":169,"props":16048,"children":16050},{"className":16049,"style":4935},[242],[],{"type":150,"tag":169,"props":16052,"children":16054},{"className":16053},[248],[16055,16060],{"type":150,"tag":169,"props":16056,"children":16058},{"className":16057,"style":2427},[248,489],[16059],{"type":155,"value":1876},{"type":150,"tag":169,"props":16061,"children":16063},{"className":16062},[495],[16064],{"type":150,"tag":169,"props":16065,"children":16067},{"className":16066},[253,254],[16068,16097],{"type":150,"tag":169,"props":16069,"children":16071},{"className":16070},[259],[16072,16092],{"type":150,"tag":169,"props":16073,"children":16075},{"className":16074,"style":802},[264],[16076],{"type":150,"tag":169,"props":16077,"children":16078},{"style":14350},[16079,16083],{"type":150,"tag":169,"props":16080,"children":16082},{"className":16081,"style":517},[274],[],{"type":150,"tag":169,"props":16084,"children":16086},{"className":16085},[522,523,524,525],[16087],{"type":150,"tag":169,"props":16088,"children":16090},{"className":16089},[248,489,525],[16091],{"type":155,"value":739},{"type":150,"tag":169,"props":16093,"children":16095},{"className":16094},[355],[16096],{"type":155,"value":358},{"type":150,"tag":169,"props":16098,"children":16100},{"className":16099},[259],[16101],{"type":150,"tag":169,"props":16102,"children":16104},{"className":16103,"style":544},[264],[16105],{"type":150,"tag":169,"props":16106,"children":16107},{},[],{"type":155,"value":16109},"。同理 ",{"type":150,"tag":169,"props":16111,"children":16113},{"className":16112},[177],[16114,16199],{"type":150,"tag":169,"props":16115,"children":16117},{"className":16116},[182],[16118],{"type":150,"tag":185,"props":16119,"children":16120},{"xmlns":187},[16121],{"type":150,"tag":191,"props":16122,"children":16123},{},[16124,16194],{"type":150,"tag":195,"props":16125,"children":16126},{},[16127,16153,16157,16168],{"type":150,"tag":1224,"props":16128,"children":16129},{},[16130,16134,16138],{"type":150,"tag":405,"props":16131,"children":16132},{},[16133],{"type":155,"value":1915},{"type":150,"tag":405,"props":16135,"children":16136},{},[16137],{"type":155,"value":739},{"type":150,"tag":195,"props":16139,"children":16140},{},[16141,16145,16149],{"type":150,"tag":417,"props":16142,"children":16143},{"stretchy":207},[16144],{"type":155,"value":972},{"type":150,"tag":405,"props":16146,"children":16147},{},[16148],{"type":155,"value":977},{"type":150,"tag":417,"props":16150,"children":16151},{"stretchy":207},[16152],{"type":155,"value":982},{"type":150,"tag":417,"props":16154,"children":16155},{},[16156],{"type":155,"value":1045},{"type":150,"tag":401,"props":16158,"children":16159},{},[16160,16164],{"type":150,"tag":405,"props":16161,"children":16162},{},[16163],{"type":155,"value":10046},{"type":150,"tag":405,"props":16165,"children":16166},{},[16167],{"type":155,"value":739},{"type":150,"tag":1224,"props":16169,"children":16170},{},[16171,16175,16179],{"type":150,"tag":405,"props":16172,"children":16173},{},[16174],{"type":155,"value":3443},{"type":150,"tag":405,"props":16176,"children":16177},{},[16178],{"type":155,"value":1915},{"type":150,"tag":195,"props":16180,"children":16181},{},[16182,16186,16190],{"type":150,"tag":417,"props":16183,"children":16184},{"stretchy":207},[16185],{"type":155,"value":972},{"type":150,"tag":405,"props":16187,"children":16188},{},[16189],{"type":155,"value":977},{"type":150,"tag":417,"props":16191,"children":16192},{"stretchy":207},[16193],{"type":155,"value":982},{"type":150,"tag":223,"props":16195,"children":16196},{"encoding":225},[16197],{"type":155,"value":16198},"v_i^{(s)} = c_i W_v^{(s)}",{"type":150,"tag":169,"props":16200,"children":16202},{"className":16201,"ariaHidden":214},[232],[16203,16311],{"type":150,"tag":169,"props":16204,"children":16206},{"className":16205},[237],[16207,16211,16298,16302,16307],{"type":150,"tag":169,"props":16208,"children":16210},{"className":16209,"style":3694},[242],[],{"type":150,"tag":169,"props":16212,"children":16214},{"className":16213},[248],[16215,16220],{"type":150,"tag":169,"props":16216,"children":16218},{"className":16217,"style":2329},[248,489],[16219],{"type":155,"value":1915},{"type":150,"tag":169,"props":16221,"children":16223},{"className":16222},[495],[16224],{"type":150,"tag":169,"props":16225,"children":16227},{"className":16226},[253,254],[16228,16287],{"type":150,"tag":169,"props":16229,"children":16231},{"className":16230},[259],[16232,16282],{"type":150,"tag":169,"props":16233,"children":16235},{"className":16234,"style":1454},[264],[16236,16252],{"type":150,"tag":169,"props":16237,"children":16238},{"style":3292},[16239,16243],{"type":150,"tag":169,"props":16240,"children":16242},{"className":16241,"style":517},[274],[],{"type":150,"tag":169,"props":16244,"children":16246},{"className":16245},[522,523,524,525],[16247],{"type":150,"tag":169,"props":16248,"children":16250},{"className":16249},[248,489,525],[16251],{"type":155,"value":739},{"type":150,"tag":169,"props":16253,"children":16254},{"style":1475},[16255,16259],{"type":150,"tag":169,"props":16256,"children":16258},{"className":16257,"style":517},[274],[],{"type":150,"tag":169,"props":16260,"children":16262},{"className":16261},[522,523,524,525],[16263],{"type":150,"tag":169,"props":16264,"children":16266},{"className":16265},[248,525],[16267,16272,16277],{"type":150,"tag":169,"props":16268,"children":16270},{"className":16269},[1004,525],[16271],{"type":155,"value":972},{"type":150,"tag":169,"props":16273,"children":16275},{"className":16274},[248,489,525],[16276],{"type":155,"value":977},{"type":150,"tag":169,"props":16278,"children":16280},{"className":16279},[1015,525],[16281],{"type":155,"value":982},{"type":150,"tag":169,"props":16283,"children":16285},{"className":16284},[355],[16286],{"type":155,"value":358},{"type":150,"tag":169,"props":16288,"children":16290},{"className":16289},[259],[16291],{"type":150,"tag":169,"props":16292,"children":16294},{"className":16293,"style":2960},[264],[16295],{"type":150,"tag":169,"props":16296,"children":16297},{},[],{"type":150,"tag":169,"props":16299,"children":16301},{"className":16300,"style":839},[558],[],{"type":150,"tag":169,"props":16303,"children":16305},{"className":16304},[844],[16306],{"type":155,"value":1045},{"type":150,"tag":169,"props":16308,"children":16310},{"className":16309,"style":839},[558],[],{"type":150,"tag":169,"props":16312,"children":16314},{"className":16313},[237],[16315,16320,16377],{"type":150,"tag":169,"props":16316,"children":16319},{"className":16317,"style":16318},[242],"height:1.1948em;vertical-align:-0.15em;",[],{"type":150,"tag":169,"props":16321,"children":16323},{"className":16322},[248],[16324,16329],{"type":150,"tag":169,"props":16325,"children":16327},{"className":16326},[248,489],[16328],{"type":155,"value":10046},{"type":150,"tag":169,"props":16330,"children":16332},{"className":16331},[495],[16333],{"type":150,"tag":169,"props":16334,"children":16336},{"className":16335},[253,254],[16337,16366],{"type":150,"tag":169,"props":16338,"children":16340},{"className":16339},[259],[16341,16361],{"type":150,"tag":169,"props":16342,"children":16344},{"className":16343,"style":802},[264],[16345],{"type":150,"tag":169,"props":16346,"children":16347},{"style":512},[16348,16352],{"type":150,"tag":169,"props":16349,"children":16351},{"className":16350,"style":517},[274],[],{"type":150,"tag":169,"props":16353,"children":16355},{"className":16354},[522,523,524,525],[16356],{"type":150,"tag":169,"props":16357,"children":16359},{"className":16358},[248,489,525],[16360],{"type":155,"value":739},{"type":150,"tag":169,"props":16362,"children":16364},{"className":16363},[355],[16365],{"type":155,"value":358},{"type":150,"tag":169,"props":16367,"children":16369},{"className":16368},[259],[16370],{"type":150,"tag":169,"props":16371,"children":16373},{"className":16372,"style":544},[264],[16374],{"type":150,"tag":169,"props":16375,"children":16376},{},[],{"type":150,"tag":169,"props":16378,"children":16380},{"className":16379},[248],[16381,16386],{"type":150,"tag":169,"props":16382,"children":16384},{"className":16383,"style":3869},[248,489],[16385],{"type":155,"value":3443},{"type":150,"tag":169,"props":16387,"children":16389},{"className":16388},[495],[16390],{"type":150,"tag":169,"props":16391,"children":16393},{"className":16392},[253,254],[16394,16453],{"type":150,"tag":169,"props":16395,"children":16397},{"className":16396},[259],[16398,16448],{"type":150,"tag":169,"props":16399,"children":16401},{"className":16400,"style":1454},[264],[16402,16418],{"type":150,"tag":169,"props":16403,"children":16404},{"style":15747},[16405,16409],{"type":150,"tag":169,"props":16406,"children":16408},{"className":16407,"style":517},[274],[],{"type":150,"tag":169,"props":16410,"children":16412},{"className":16411},[522,523,524,525],[16413],{"type":150,"tag":169,"props":16414,"children":16416},{"className":16415,"style":2329},[248,489,525],[16417],{"type":155,"value":1915},{"type":150,"tag":169,"props":16419,"children":16420},{"style":1475},[16421,16425],{"type":150,"tag":169,"props":16422,"children":16424},{"className":16423,"style":517},[274],[],{"type":150,"tag":169,"props":16426,"children":16428},{"className":16427},[522,523,524,525],[16429],{"type":150,"tag":169,"props":16430,"children":16432},{"className":16431},[248,525],[16433,16438,16443],{"type":150,"tag":169,"props":16434,"children":16436},{"className":16435},[1004,525],[16437],{"type":155,"value":972},{"type":150,"tag":169,"props":16439,"children":16441},{"className":16440},[248,489,525],[16442],{"type":155,"value":977},{"type":150,"tag":169,"props":16444,"children":16446},{"className":16445},[1015,525],[16447],{"type":155,"value":982},{"type":150,"tag":169,"props":16449,"children":16451},{"className":16450},[355],[16452],{"type":155,"value":358},{"type":150,"tag":169,"props":16454,"children":16456},{"className":16455},[259],[16457],{"type":150,"tag":169,"props":16458,"children":16461},{"className":16459,"style":16460},[264],"height:0.1166em;",[16462],{"type":150,"tag":169,"props":16463,"children":16464},{},[],{"type":155,"value":16466}," 里的 ",{"type":150,"tag":169,"props":16468,"children":16470},{"className":16469},[177],[16471,16515],{"type":150,"tag":169,"props":16472,"children":16474},{"className":16473},[182],[16475],{"type":150,"tag":185,"props":16476,"children":16477},{"xmlns":187},[16478],{"type":150,"tag":191,"props":16479,"children":16480},{},[16481,16510],{"type":150,"tag":195,"props":16482,"children":16483},{},[16484],{"type":150,"tag":1224,"props":16485,"children":16486},{},[16487,16491,16495],{"type":150,"tag":405,"props":16488,"children":16489},{},[16490],{"type":155,"value":3443},{"type":150,"tag":405,"props":16492,"children":16493},{},[16494],{"type":155,"value":1915},{"type":150,"tag":195,"props":16496,"children":16497},{},[16498,16502,16506],{"type":150,"tag":417,"props":16499,"children":16500},{"stretchy":207},[16501],{"type":155,"value":972},{"type":150,"tag":405,"props":16503,"children":16504},{},[16505],{"type":155,"value":977},{"type":150,"tag":417,"props":16507,"children":16508},{"stretchy":207},[16509],{"type":155,"value":982},{"type":150,"tag":223,"props":16511,"children":16512},{"encoding":225},[16513],{"type":155,"value":16514},"W_v^{(s)}",{"type":150,"tag":169,"props":16516,"children":16518},{"className":16517,"ariaHidden":214},[232],[16519],{"type":150,"tag":169,"props":16520,"children":16522},{"className":16521},[237],[16523,16528],{"type":150,"tag":169,"props":16524,"children":16527},{"className":16525,"style":16526},[242],"height:1.1614em;vertical-align:-0.1166em;",[],{"type":150,"tag":169,"props":16529,"children":16531},{"className":16530},[248],[16532,16537],{"type":150,"tag":169,"props":16533,"children":16535},{"className":16534,"style":3869},[248,489],[16536],{"type":155,"value":3443},{"type":150,"tag":169,"props":16538,"children":16540},{"className":16539},[495],[16541],{"type":150,"tag":169,"props":16542,"children":16544},{"className":16543},[253,254],[16545,16604],{"type":150,"tag":169,"props":16546,"children":16548},{"className":16547},[259],[16549,16599],{"type":150,"tag":169,"props":16550,"children":16552},{"className":16551,"style":1454},[264],[16553,16569],{"type":150,"tag":169,"props":16554,"children":16555},{"style":15747},[16556,16560],{"type":150,"tag":169,"props":16557,"children":16559},{"className":16558,"style":517},[274],[],{"type":150,"tag":169,"props":16561,"children":16563},{"className":16562},[522,523,524,525],[16564],{"type":150,"tag":169,"props":16565,"children":16567},{"className":16566,"style":2329},[248,489,525],[16568],{"type":155,"value":1915},{"type":150,"tag":169,"props":16570,"children":16571},{"style":1475},[16572,16576],{"type":150,"tag":169,"props":16573,"children":16575},{"className":16574,"style":517},[274],[],{"type":150,"tag":169,"props":16577,"children":16579},{"className":16578},[522,523,524,525],[16580],{"type":150,"tag":169,"props":16581,"children":16583},{"className":16582},[248,525],[16584,16589,16594],{"type":150,"tag":169,"props":16585,"children":16587},{"className":16586},[1004,525],[16588],{"type":155,"value":972},{"type":150,"tag":169,"props":16590,"children":16592},{"className":16591},[248,489,525],[16593],{"type":155,"value":977},{"type":150,"tag":169,"props":16595,"children":16597},{"className":16596},[1015,525],[16598],{"type":155,"value":982},{"type":150,"tag":169,"props":16600,"children":16602},{"className":16601},[355],[16603],{"type":155,"value":358},{"type":150,"tag":169,"props":16605,"children":16607},{"className":16606},[259],[16608],{"type":150,"tag":169,"props":16609,"children":16611},{"className":16610,"style":16460},[264],[16612],{"type":150,"tag":169,"props":16613,"children":16614},{},[],{"type":155,"value":16616}," 也能吸收进后面的输出投影，",{"type":150,"tag":169,"props":16618,"children":16620},{"className":16619},[177],[16621,16650],{"type":150,"tag":169,"props":16622,"children":16624},{"className":16623},[182],[16625],{"type":150,"tag":185,"props":16626,"children":16627},{"xmlns":187},[16628],{"type":150,"tag":191,"props":16629,"children":16630},{},[16631,16645],{"type":150,"tag":195,"props":16632,"children":16633},{},[16634],{"type":150,"tag":401,"props":16635,"children":16636},{},[16637,16641],{"type":150,"tag":405,"props":16638,"children":16639},{},[16640],{"type":155,"value":1915},{"type":150,"tag":405,"props":16642,"children":16643},{},[16644],{"type":155,"value":739},{"type":150,"tag":223,"props":16646,"children":16647},{"encoding":225},[16648],{"type":155,"value":16649},"v_i",{"type":150,"tag":169,"props":16651,"children":16653},{"className":16652,"ariaHidden":214},[232],[16654],{"type":150,"tag":169,"props":16655,"children":16657},{"className":16656},[237],[16658,16662],{"type":150,"tag":169,"props":16659,"children":16661},{"className":16660,"style":1343},[242],[],{"type":150,"tag":169,"props":16663,"children":16665},{"className":16664},[248],[16666,16671],{"type":150,"tag":169,"props":16667,"children":16669},{"className":16668,"style":2329},[248,489],[16670],{"type":155,"value":1915},{"type":150,"tag":169,"props":16672,"children":16674},{"className":16673},[495],[16675],{"type":150,"tag":169,"props":16676,"children":16678},{"className":16677},[253,254],[16679,16708],{"type":150,"tag":169,"props":16680,"children":16682},{"className":16681},[259],[16683,16703],{"type":150,"tag":169,"props":16684,"children":16686},{"className":16685,"style":802},[264],[16687],{"type":150,"tag":169,"props":16688,"children":16689},{"style":14417},[16690,16694],{"type":150,"tag":169,"props":16691,"children":16693},{"className":16692,"style":517},[274],[],{"type":150,"tag":169,"props":16695,"children":16697},{"className":16696},[522,523,524,525],[16698],{"type":150,"tag":169,"props":16699,"children":16701},{"className":16700},[248,489,525],[16702],{"type":155,"value":739},{"type":150,"tag":169,"props":16704,"children":16706},{"className":16705},[355],[16707],{"type":155,"value":358},{"type":150,"tag":169,"props":16709,"children":16711},{"className":16710},[259],[16712],{"type":150,"tag":169,"props":16713,"children":16715},{"className":16714,"style":544},[264],[16716],{"type":150,"tag":169,"props":16717,"children":16718},{},[],{"type":155,"value":16720}," 也可以用 ",{"type":150,"tag":169,"props":16722,"children":16724},{"className":16723},[177],[16725,16753],{"type":150,"tag":169,"props":16726,"children":16728},{"className":16727},[182],[16729],{"type":150,"tag":185,"props":16730,"children":16731},{"xmlns":187},[16732],{"type":150,"tag":191,"props":16733,"children":16734},{},[16735,16749],{"type":150,"tag":195,"props":16736,"children":16737},{},[16738],{"type":150,"tag":401,"props":16739,"children":16740},{},[16741,16745],{"type":150,"tag":405,"props":16742,"children":16743},{},[16744],{"type":155,"value":10046},{"type":150,"tag":405,"props":16746,"children":16747},{},[16748],{"type":155,"value":739},{"type":150,"tag":223,"props":16750,"children":16751},{"encoding":225},[16752],{"type":155,"value":10055},{"type":150,"tag":169,"props":16754,"children":16756},{"className":16755,"ariaHidden":214},[232],[16757],{"type":150,"tag":169,"props":16758,"children":16760},{"className":16759},[237],[16761,16765],{"type":150,"tag":169,"props":16762,"children":16764},{"className":16763,"style":1343},[242],[],{"type":150,"tag":169,"props":16766,"children":16768},{"className":16767},[248],[16769,16774],{"type":150,"tag":169,"props":16770,"children":16772},{"className":16771},[248,489],[16773],{"type":155,"value":10046},{"type":150,"tag":169,"props":16775,"children":16777},{"className":16776},[495],[16778],{"type":150,"tag":169,"props":16779,"children":16781},{"className":16780},[253,254],[16782,16811],{"type":150,"tag":169,"props":16783,"children":16785},{"className":16784},[259],[16786,16806],{"type":150,"tag":169,"props":16787,"children":16789},{"className":16788,"style":802},[264],[16790],{"type":150,"tag":169,"props":16791,"children":16792},{"style":512},[16793,16797],{"type":150,"tag":169,"props":16794,"children":16796},{"className":16795,"style":517},[274],[],{"type":150,"tag":169,"props":16798,"children":16800},{"className":16799},[522,523,524,525],[16801],{"type":150,"tag":169,"props":16802,"children":16804},{"className":16803},[248,489,525],[16805],{"type":155,"value":739},{"type":150,"tag":169,"props":16807,"children":16809},{"className":16808},[355],[16810],{"type":155,"value":358},{"type":150,"tag":169,"props":16812,"children":16814},{"className":16813},[259],[16815],{"type":150,"tag":169,"props":16816,"children":16818},{"className":16817,"style":544},[264],[16819],{"type":150,"tag":169,"props":16820,"children":16821},{},[],{"type":155,"value":16823}," 代替。于是推理时 KV Cache 只需要存 ",{"type":150,"tag":169,"props":16825,"children":16827},{"className":16826},[177],[16828,16856],{"type":150,"tag":169,"props":16829,"children":16831},{"className":16830},[182],[16832],{"type":150,"tag":185,"props":16833,"children":16834},{"xmlns":187},[16835],{"type":150,"tag":191,"props":16836,"children":16837},{},[16838,16852],{"type":150,"tag":195,"props":16839,"children":16840},{},[16841],{"type":150,"tag":401,"props":16842,"children":16843},{},[16844,16848],{"type":150,"tag":405,"props":16845,"children":16846},{},[16847],{"type":155,"value":10046},{"type":150,"tag":405,"props":16849,"children":16850},{},[16851],{"type":155,"value":739},{"type":150,"tag":223,"props":16853,"children":16854},{"encoding":225},[16855],{"type":155,"value":10055},{"type":150,"tag":169,"props":16857,"children":16859},{"className":16858,"ariaHidden":214},[232],[16860],{"type":150,"tag":169,"props":16861,"children":16863},{"className":16862},[237],[16864,16868],{"type":150,"tag":169,"props":16865,"children":16867},{"className":16866,"style":1343},[242],[],{"type":150,"tag":169,"props":16869,"children":16871},{"className":16870},[248],[16872,16877],{"type":150,"tag":169,"props":16873,"children":16875},{"className":16874},[248,489],[16876],{"type":155,"value":10046},{"type":150,"tag":169,"props":16878,"children":16880},{"className":16879},[495],[16881],{"type":150,"tag":169,"props":16882,"children":16884},{"className":16883},[253,254],[16885,16914],{"type":150,"tag":169,"props":16886,"children":16888},{"className":16887},[259],[16889,16909],{"type":150,"tag":169,"props":16890,"children":16892},{"className":16891,"style":802},[264],[16893],{"type":150,"tag":169,"props":16894,"children":16895},{"style":512},[16896,16900],{"type":150,"tag":169,"props":16897,"children":16899},{"className":16898,"style":517},[274],[],{"type":150,"tag":169,"props":16901,"children":16903},{"className":16902},[522,523,524,525],[16904],{"type":150,"tag":169,"props":16905,"children":16907},{"className":16906},[248,489,525],[16908],{"type":155,"value":739},{"type":150,"tag":169,"props":16910,"children":16912},{"className":16911},[355],[16913],{"type":155,"value":358},{"type":150,"tag":169,"props":16915,"children":16917},{"className":16916},[259],[16918],{"type":150,"tag":169,"props":16919,"children":16921},{"className":16920,"style":544},[264],[16922],{"type":150,"tag":169,"props":16923,"children":16924},{},[],{"type":155,"value":16926},"。而 ",{"type":150,"tag":169,"props":16928,"children":16930},{"className":16929},[177],[16931,16959],{"type":150,"tag":169,"props":16932,"children":16934},{"className":16933},[182],[16935],{"type":150,"tag":185,"props":16936,"children":16937},{"xmlns":187},[16938],{"type":150,"tag":191,"props":16939,"children":16940},{},[16941,16955],{"type":150,"tag":195,"props":16942,"children":16943},{},[16944],{"type":150,"tag":401,"props":16945,"children":16946},{},[16947,16951],{"type":150,"tag":405,"props":16948,"children":16949},{},[16950],{"type":155,"value":10046},{"type":150,"tag":405,"props":16952,"children":16953},{},[16954],{"type":155,"value":739},{"type":150,"tag":223,"props":16956,"children":16957},{"encoding":225},[16958],{"type":155,"value":10055},{"type":150,"tag":169,"props":16960,"children":16962},{"className":16961,"ariaHidden":214},[232],[16963],{"type":150,"tag":169,"props":16964,"children":16966},{"className":16965},[237],[16967,16971],{"type":150,"tag":169,"props":16968,"children":16970},{"className":16969,"style":1343},[242],[],{"type":150,"tag":169,"props":16972,"children":16974},{"className":16973},[248],[16975,16980],{"type":150,"tag":169,"props":16976,"children":16978},{"className":16977},[248,489],[16979],{"type":155,"value":10046},{"type":150,"tag":169,"props":16981,"children":16983},{"className":16982},[495],[16984],{"type":150,"tag":169,"props":16985,"children":16987},{"className":16986},[253,254],[16988,17017],{"type":150,"tag":169,"props":16989,"children":16991},{"className":16990},[259],[16992,17012],{"type":150,"tag":169,"props":16993,"children":16995},{"className":16994,"style":802},[264],[16996],{"type":150,"tag":169,"props":16997,"children":16998},{"style":512},[16999,17003],{"type":150,"tag":169,"props":17000,"children":17002},{"className":17001,"style":517},[274],[],{"type":150,"tag":169,"props":17004,"children":17006},{"className":17005},[522,523,524,525],[17007],{"type":150,"tag":169,"props":17008,"children":17010},{"className":17009},[248,489,525],[17011],{"type":155,"value":739},{"type":150,"tag":169,"props":17013,"children":17015},{"className":17014},[355],[17016],{"type":155,"value":358},{"type":150,"tag":169,"props":17018,"children":17020},{"className":17019},[259],[17021],{"type":150,"tag":169,"props":17022,"children":17024},{"className":17023,"style":544},[264],[17025],{"type":150,"tag":169,"props":17026,"children":17027},{},[],{"type":155,"value":17029}," 与 ",{"type":150,"tag":169,"props":17031,"children":17033},{"className":17032},[177],[17034,17063],{"type":150,"tag":169,"props":17035,"children":17037},{"className":17036},[182],[17038],{"type":150,"tag":185,"props":17039,"children":17040},{"xmlns":187},[17041],{"type":150,"tag":191,"props":17042,"children":17043},{},[17044,17059],{"type":150,"tag":195,"props":17045,"children":17046},{},[17047,17051,17055],{"type":150,"tag":417,"props":17048,"children":17049},{"stretchy":207},[17050],{"type":155,"value":972},{"type":150,"tag":405,"props":17052,"children":17053},{},[17054],{"type":155,"value":977},{"type":150,"tag":417,"props":17056,"children":17057},{"stretchy":207},[17058],{"type":155,"value":982},{"type":150,"tag":223,"props":17060,"children":17061},{"encoding":225},[17062],{"type":155,"value":987},{"type":150,"tag":169,"props":17064,"children":17066},{"className":17065,"ariaHidden":214},[232],[17067],{"type":150,"tag":169,"props":17068,"children":17070},{"className":17069},[237],[17071,17075,17080,17085],{"type":150,"tag":169,"props":17072,"children":17074},{"className":17073,"style":999},[242],[],{"type":150,"tag":169,"props":17076,"children":17078},{"className":17077},[1004],[17079],{"type":155,"value":972},{"type":150,"tag":169,"props":17081,"children":17083},{"className":17082},[248,489],[17084],{"type":155,"value":977},{"type":150,"tag":169,"props":17086,"children":17088},{"className":17087},[1015],[17089],{"type":155,"value":982},{"type":155,"value":17091}," 无关，是所有 head 共享的。",{"type":150,"tag":151,"props":17093,"children":17094},{},[17095,17100,17102,17204,17206,17428],{"type":150,"tag":158,"props":17096,"children":17097},{},[17098],{"type":155,"value":17099},"MLA 在推理阶段恒等变换成了一个 MQA。",{"type":155,"value":17101}," 到这里 MLA 完成了什么就很清楚了：它用不同的投影矩阵增强了 GQA 的能力，同时在推理时保持同样大小的 KV Cache。反过来想，如果只需要跟 GQA 相当的能力，那 ",{"type":150,"tag":169,"props":17103,"children":17105},{"className":17104},[177],[17106,17135],{"type":150,"tag":169,"props":17107,"children":17109},{"className":17108},[182],[17110],{"type":150,"tag":185,"props":17111,"children":17112},{"xmlns":187},[17113],{"type":150,"tag":191,"props":17114,"children":17115},{},[17116,17130],{"type":150,"tag":195,"props":17117,"children":17118},{},[17119],{"type":150,"tag":401,"props":17120,"children":17121},{},[17122,17126],{"type":150,"tag":405,"props":17123,"children":17124},{},[17125],{"type":155,"value":759},{"type":150,"tag":405,"props":17127,"children":17128},{},[17129],{"type":155,"value":10046},{"type":150,"tag":223,"props":17131,"children":17132},{"encoding":225},[17133],{"type":155,"value":17134},"d_c",{"type":150,"tag":169,"props":17136,"children":17138},{"className":17137,"ariaHidden":214},[232],[17139],{"type":150,"tag":169,"props":17140,"children":17142},{"className":17141},[237],[17143,17147],{"type":150,"tag":169,"props":17144,"children":17146},{"className":17145,"style":4935},[242],[],{"type":150,"tag":169,"props":17148,"children":17150},{"className":17149},[248],[17151,17156],{"type":150,"tag":169,"props":17152,"children":17154},{"className":17153},[248,489],[17155],{"type":155,"value":759},{"type":150,"tag":169,"props":17157,"children":17159},{"className":17158},[495],[17160],{"type":150,"tag":169,"props":17161,"children":17163},{"className":17162},[253,254],[17164,17193],{"type":150,"tag":169,"props":17165,"children":17167},{"className":17166},[259],[17168,17188],{"type":150,"tag":169,"props":17169,"children":17171},{"className":17170,"style":5039},[264],[17172],{"type":150,"tag":169,"props":17173,"children":17174},{"style":512},[17175,17179],{"type":150,"tag":169,"props":17176,"children":17178},{"className":17177,"style":517},[274],[],{"type":150,"tag":169,"props":17180,"children":17182},{"className":17181},[522,523,524,525],[17183],{"type":150,"tag":169,"props":17184,"children":17186},{"className":17185},[248,489,525],[17187],{"type":155,"value":10046},{"type":150,"tag":169,"props":17189,"children":17191},{"className":17190},[355],[17192],{"type":155,"value":358},{"type":150,"tag":169,"props":17194,"children":17196},{"className":17195},[259],[17197],{"type":150,"tag":169,"props":17198,"children":17200},{"className":17199,"style":544},[264],[17201],{"type":150,"tag":169,"props":17202,"children":17203},{},[],{"type":155,"value":17205}," 就不必取 ",{"type":150,"tag":169,"props":17207,"children":17209},{"className":17208},[177],[17210,17266],{"type":150,"tag":169,"props":17211,"children":17213},{"className":17212},[182],[17214],{"type":150,"tag":185,"props":17215,"children":17216},{"xmlns":187},[17217],{"type":150,"tag":191,"props":17218,"children":17219},{},[17220,17261],{"type":150,"tag":195,"props":17221,"children":17222},{},[17223,17227,17231,17242,17246,17257],{"type":150,"tag":405,"props":17224,"children":17225},{},[17226],{"type":155,"value":8100},{"type":150,"tag":417,"props":17228,"children":17229},{"stretchy":207},[17230],{"type":155,"value":972},{"type":150,"tag":401,"props":17232,"children":17233},{},[17234,17238],{"type":150,"tag":405,"props":17235,"children":17236},{},[17237],{"type":155,"value":759},{"type":150,"tag":405,"props":17239,"children":17240},{},[17241],{"type":155,"value":1876},{"type":150,"tag":417,"props":17243,"children":17244},{},[17245],{"type":155,"value":5136},{"type":150,"tag":401,"props":17247,"children":17248},{},[17249,17253],{"type":150,"tag":405,"props":17250,"children":17251},{},[17252],{"type":155,"value":759},{"type":150,"tag":405,"props":17254,"children":17255},{},[17256],{"type":155,"value":1915},{"type":150,"tag":417,"props":17258,"children":17259},{"stretchy":207},[17260],{"type":155,"value":982},{"type":150,"tag":223,"props":17262,"children":17263},{"encoding":225},[17264],{"type":155,"value":17265},"g(d_k + d_v)",{"type":150,"tag":169,"props":17267,"children":17269},{"className":17268,"ariaHidden":214},[232],[17270,17358],{"type":150,"tag":169,"props":17271,"children":17273},{"className":17272},[237],[17274,17278,17283,17288,17345,17349,17354],{"type":150,"tag":169,"props":17275,"children":17277},{"className":17276,"style":999},[242],[],{"type":150,"tag":169,"props":17279,"children":17281},{"className":17280,"style":2329},[248,489],[17282],{"type":155,"value":8100},{"type":150,"tag":169,"props":17284,"children":17286},{"className":17285},[1004],[17287],{"type":155,"value":972},{"type":150,"tag":169,"props":17289,"children":17291},{"className":17290},[248],[17292,17297],{"type":150,"tag":169,"props":17293,"children":17295},{"className":17294},[248,489],[17296],{"type":155,"value":759},{"type":150,"tag":169,"props":17298,"children":17300},{"className":17299},[495],[17301],{"type":150,"tag":169,"props":17302,"children":17304},{"className":17303},[253,254],[17305,17334],{"type":150,"tag":169,"props":17306,"children":17308},{"className":17307},[259],[17309,17329],{"type":150,"tag":169,"props":17310,"children":17312},{"className":17311,"style":674},[264],[17313],{"type":150,"tag":169,"props":17314,"children":17315},{"style":512},[17316,17320],{"type":150,"tag":169,"props":17317,"children":17319},{"className":17318,"style":517},[274],[],{"type":150,"tag":169,"props":17321,"children":17323},{"className":17322},[522,523,524,525],[17324],{"type":150,"tag":169,"props":17325,"children":17327},{"className":17326,"style":2427},[248,489,525],[17328],{"type":155,"value":1876},{"type":150,"tag":169,"props":17330,"children":17332},{"className":17331},[355],[17333],{"type":155,"value":358},{"type":150,"tag":169,"props":17335,"children":17337},{"className":17336},[259],[17338],{"type":150,"tag":169,"props":17339,"children":17341},{"className":17340,"style":544},[264],[17342],{"type":150,"tag":169,"props":17343,"children":17344},{},[],{"type":150,"tag":169,"props":17346,"children":17348},{"className":17347,"style":5167},[558],[],{"type":150,"tag":169,"props":17350,"children":17352},{"className":17351},[5172],[17353],{"type":155,"value":5136},{"type":150,"tag":169,"props":17355,"children":17357},{"className":17356,"style":5167},[558],[],{"type":150,"tag":169,"props":17359,"children":17361},{"className":17360},[237],[17362,17366,17423],{"type":150,"tag":169,"props":17363,"children":17365},{"className":17364,"style":999},[242],[],{"type":150,"tag":169,"props":17367,"children":17369},{"className":17368},[248],[17370,17375],{"type":150,"tag":169,"props":17371,"children":17373},{"className":17372},[248,489],[17374],{"type":155,"value":759},{"type":150,"tag":169,"props":17376,"children":17378},{"className":17377},[495],[17379],{"type":150,"tag":169,"props":17380,"children":17382},{"className":17381},[253,254],[17383,17412],{"type":150,"tag":169,"props":17384,"children":17386},{"className":17385},[259],[17387,17407],{"type":150,"tag":169,"props":17388,"children":17390},{"className":17389,"style":5039},[264],[17391],{"type":150,"tag":169,"props":17392,"children":17393},{"style":512},[17394,17398],{"type":150,"tag":169,"props":17395,"children":17397},{"className":17396,"style":517},[274],[],{"type":150,"tag":169,"props":17399,"children":17401},{"className":17400},[522,523,524,525],[17402],{"type":150,"tag":169,"props":17403,"children":17405},{"className":17404,"style":2329},[248,489,525],[17406],{"type":155,"value":1915},{"type":150,"tag":169,"props":17408,"children":17410},{"className":17409},[355],[17411],{"type":155,"value":358},{"type":150,"tag":169,"props":17413,"children":17415},{"className":17414},[259],[17416],{"type":150,"tag":169,"props":17417,"children":17419},{"className":17418,"style":544},[264],[17420],{"type":150,"tag":169,"props":17421,"children":17422},{},[],{"type":150,"tag":169,"props":17424,"children":17426},{"className":17425},[1015],[17427],{"type":155,"value":982},{"type":155,"value":17429},"，取更小的值就行。DeepSeek-V2 取了 512。",{"type":150,"tag":151,"props":17431,"children":17432},{},[17433],{"type":155,"value":17434},"这就是 MLA 的核心。低秩压缩为什么有效？因为不同 head 需要的信息本来就有大量重叠。下投影矩阵负责提取这些共同信息，上投影矩阵让各个 head 用各自的方式去使用它。这甚至可能带来一点正则化效果。它强制模型把冗余的共同信息压进同一个低维空间，而不是每个 head 各存一份。",{"type":150,"tag":151,"props":17436,"children":17437},{},[17438],{"type":155,"value":17439},"我一开始以为 MLA 只是\"把 KV 存小一点\"。后来才反应过来，它换掉的是一整套共享方式。存小只是结果，不是原因。",{"type":150,"tag":17441,"props":17442,"children":17443},"blockquote",{},[17444,17452,17725],{"type":150,"tag":151,"props":17445,"children":17446},{},[17447],{"type":150,"tag":158,"props":17448,"children":17449},{},[17450],{"type":155,"value":17451},"补充说明",{"type":150,"tag":151,"props":17453,"children":17454},{},[17455,17723],{"type":150,"tag":169,"props":17456,"children":17458},{"className":17457},[177],[17459,17532],{"type":150,"tag":169,"props":17460,"children":17462},{"className":17461},[182],[17463],{"type":150,"tag":185,"props":17464,"children":17465},{"xmlns":187},[17466],{"type":150,"tag":191,"props":17467,"children":17468},{},[17469,17528],{"type":150,"tag":195,"props":17470,"children":17471},{},[17472,17498],{"type":150,"tag":1224,"props":17473,"children":17474},{},[17475,17479,17483],{"type":150,"tag":405,"props":17476,"children":17477},{},[17478],{"type":155,"value":3443},{"type":150,"tag":405,"props":17480,"children":17481},{},[17482],{"type":155,"value":1845},{"type":150,"tag":195,"props":17484,"children":17485},{},[17486,17490,17494],{"type":150,"tag":417,"props":17487,"children":17488},{"stretchy":207},[17489],{"type":155,"value":972},{"type":150,"tag":405,"props":17491,"children":17492},{},[17493],{"type":155,"value":977},{"type":150,"tag":417,"props":17495,"children":17496},{"stretchy":207},[17497],{"type":155,"value":982},{"type":150,"tag":1224,"props":17499,"children":17500},{},[17501,17505,17509],{"type":150,"tag":405,"props":17502,"children":17503},{},[17504],{"type":155,"value":3443},{"type":150,"tag":405,"props":17506,"children":17507},{},[17508],{"type":155,"value":1876},{"type":150,"tag":195,"props":17510,"children":17511},{},[17512,17516,17520,17524],{"type":150,"tag":417,"props":17513,"children":17514},{"stretchy":207},[17515],{"type":155,"value":972},{"type":150,"tag":405,"props":17517,"children":17518},{},[17519],{"type":155,"value":977},{"type":150,"tag":417,"props":17521,"children":17522},{"stretchy":207},[17523],{"type":155,"value":982},{"type":150,"tag":405,"props":17525,"children":17526},{"mathvariant":1796},[17527],{"type":155,"value":2052},{"type":150,"tag":223,"props":17529,"children":17530},{"encoding":225},[17531],{"type":155,"value":15707},{"type":150,"tag":169,"props":17533,"children":17535},{"className":17534,"ariaHidden":214},[232],[17536],{"type":150,"tag":169,"props":17537,"children":17539},{"className":17538},[237],[17540,17544,17631],{"type":150,"tag":169,"props":17541,"children":17543},{"className":17542,"style":4200},[242],[],{"type":150,"tag":169,"props":17545,"children":17547},{"className":17546},[248],[17548,17553],{"type":150,"tag":169,"props":17549,"children":17551},{"className":17550,"style":3869},[248,489],[17552],{"type":155,"value":3443},{"type":150,"tag":169,"props":17554,"children":17556},{"className":17555},[495],[17557],{"type":150,"tag":169,"props":17558,"children":17560},{"className":17559},[253,254],[17561,17620],{"type":150,"tag":169,"props":17562,"children":17564},{"className":17563},[259],[17565,17615],{"type":150,"tag":169,"props":17566,"children":17568},{"className":17567,"style":1454},[264],[17569,17585],{"type":150,"tag":169,"props":17570,"children":17571},{"style":15747},[17572,17576],{"type":150,"tag":169,"props":17573,"children":17575},{"className":17574,"style":517},[274],[],{"type":150,"tag":169,"props":17577,"children":17579},{"className":17578},[522,523,524,525],[17580],{"type":150,"tag":169,"props":17581,"children":17583},{"className":17582,"style":2329},[248,489,525],[17584],{"type":155,"value":1845},{"type":150,"tag":169,"props":17586,"children":17587},{"style":1475},[17588,17592],{"type":150,"tag":169,"props":17589,"children":17591},{"className":17590,"style":517},[274],[],{"type":150,"tag":169,"props":17593,"children":17595},{"className":17594},[522,523,524,525],[17596],{"type":150,"tag":169,"props":17597,"children":17599},{"className":17598},[248,525],[17600,17605,17610],{"type":150,"tag":169,"props":17601,"children":17603},{"className":17602},[1004,525],[17604],{"type":155,"value":972},{"type":150,"tag":169,"props":17606,"children":17608},{"className":17607},[248,489,525],[17609],{"type":155,"value":977},{"type":150,"tag":169,"props":17611,"children":17613},{"className":17612},[1015,525],[17614],{"type":155,"value":982},{"type":150,"tag":169,"props":17616,"children":17618},{"className":17617},[355],[17619],{"type":155,"value":358},{"type":150,"tag":169,"props":17621,"children":17623},{"className":17622},[259],[17624],{"type":150,"tag":169,"props":17625,"children":17627},{"className":17626,"style":15804},[264],[17628],{"type":150,"tag":169,"props":17629,"children":17630},{},[],{"type":150,"tag":169,"props":17632,"children":17634},{"className":17633},[248],[17635,17640],{"type":150,"tag":169,"props":17636,"children":17638},{"className":17637,"style":3869},[248,489],[17639],{"type":155,"value":3443},{"type":150,"tag":169,"props":17641,"children":17643},{"className":17642},[495],[17644],{"type":150,"tag":169,"props":17645,"children":17647},{"className":17646},[253,254],[17648,17712],{"type":150,"tag":169,"props":17649,"children":17651},{"className":17650},[259],[17652,17707],{"type":150,"tag":169,"props":17653,"children":17655},{"className":17654,"style":1454},[264],[17656,17672],{"type":150,"tag":169,"props":17657,"children":17658},{"style":4286},[17659,17663],{"type":150,"tag":169,"props":17660,"children":17662},{"className":17661,"style":517},[274],[],{"type":150,"tag":169,"props":17664,"children":17666},{"className":17665},[522,523,524,525],[17667],{"type":150,"tag":169,"props":17668,"children":17670},{"className":17669,"style":2427},[248,489,525],[17671],{"type":155,"value":1876},{"type":150,"tag":169,"props":17673,"children":17674},{"style":1475},[17675,17679],{"type":150,"tag":169,"props":17676,"children":17678},{"className":17677,"style":517},[274],[],{"type":150,"tag":169,"props":17680,"children":17682},{"className":17681},[522,523,524,525],[17683],{"type":150,"tag":169,"props":17684,"children":17686},{"className":17685},[248,525],[17687,17692,17697,17702],{"type":150,"tag":169,"props":17688,"children":17690},{"className":17689},[1004,525],[17691],{"type":155,"value":972},{"type":150,"tag":169,"props":17693,"children":17695},{"className":17694},[248,489,525],[17696],{"type":155,"value":977},{"type":150,"tag":169,"props":17698,"children":17700},{"className":17699},[1015,525],[17701],{"type":155,"value":982},{"type":150,"tag":169,"props":17703,"children":17705},{"className":17704},[248,525],[17706],{"type":155,"value":2052},{"type":150,"tag":169,"props":17708,"children":17710},{"className":17709},[355],[17711],{"type":155,"value":358},{"type":150,"tag":169,"props":17713,"children":17715},{"className":17714},[259],[17716],{"type":150,"tag":169,"props":17717,"children":17719},{"className":17718,"style":4343},[264],[17720],{"type":150,"tag":169,"props":17721,"children":17722},{},[],{"type":155,"value":17724}," 合并成一个矩阵，理论上只在无限精度下成立。实际上如果用 BF16，变换后的精度损失往往挺明显，多层累积后可能放大到可观的程度。",{"type":150,"tag":151,"props":17726,"children":17727},{},[17728,17730,18085,18087,18442],{"type":155,"value":17729},"另外实际实现一般不按 ",{"type":150,"tag":169,"props":17731,"children":17733},{"className":17732},[177],[17734,17827],{"type":150,"tag":169,"props":17735,"children":17737},{"className":17736},[182],[17738],{"type":150,"tag":185,"props":17739,"children":17740},{"xmlns":187},[17741],{"type":150,"tag":191,"props":17742,"children":17743},{},[17744,17822],{"type":150,"tag":195,"props":17745,"children":17746},{},[17747,17758,17762,17788,17818],{"type":150,"tag":401,"props":17748,"children":17749},{},[17750,17754],{"type":150,"tag":405,"props":17751,"children":17752},{},[17753],{"type":155,"value":409},{"type":150,"tag":405,"props":17755,"children":17756},{},[17757],{"type":155,"value":1213},{"type":150,"tag":417,"props":17759,"children":17760},{"stretchy":207},[17761],{"type":155,"value":972},{"type":150,"tag":1224,"props":17763,"children":17764},{},[17765,17769,17773],{"type":150,"tag":405,"props":17766,"children":17767},{},[17768],{"type":155,"value":3443},{"type":150,"tag":405,"props":17770,"children":17771},{},[17772],{"type":155,"value":1845},{"type":150,"tag":195,"props":17774,"children":17775},{},[17776,17780,17784],{"type":150,"tag":417,"props":17777,"children":17778},{"stretchy":207},[17779],{"type":155,"value":972},{"type":150,"tag":405,"props":17781,"children":17782},{},[17783],{"type":155,"value":977},{"type":150,"tag":417,"props":17785,"children":17786},{"stretchy":207},[17787],{"type":155,"value":982},{"type":150,"tag":1224,"props":17789,"children":17790},{},[17791,17795,17799],{"type":150,"tag":405,"props":17792,"children":17793},{},[17794],{"type":155,"value":3443},{"type":150,"tag":405,"props":17796,"children":17797},{},[17798],{"type":155,"value":1876},{"type":150,"tag":195,"props":17800,"children":17801},{},[17802,17806,17810,17814],{"type":150,"tag":417,"props":17803,"children":17804},{"stretchy":207},[17805],{"type":155,"value":972},{"type":150,"tag":405,"props":17807,"children":17808},{},[17809],{"type":155,"value":977},{"type":150,"tag":417,"props":17811,"children":17812},{"stretchy":207},[17813],{"type":155,"value":982},{"type":150,"tag":405,"props":17815,"children":17816},{"mathvariant":1796},[17817],{"type":155,"value":2052},{"type":150,"tag":417,"props":17819,"children":17820},{"stretchy":207},[17821],{"type":155,"value":982},{"type":150,"tag":223,"props":17823,"children":17824},{"encoding":225},[17825],{"type":155,"value":17826},"x_t (W_q^{(s)} W_k^{(s)\\top})",{"type":150,"tag":169,"props":17828,"children":17830},{"className":17829,"ariaHidden":214},[232],[17831],{"type":150,"tag":169,"props":17832,"children":17834},{"className":17833},[237],[17835,17839,17896,17901,17988,18080],{"type":150,"tag":169,"props":17836,"children":17838},{"className":17837,"style":4200},[242],[],{"type":150,"tag":169,"props":17840,"children":17842},{"className":17841},[248],[17843,17848],{"type":150,"tag":169,"props":17844,"children":17846},{"className":17845},[248,489],[17847],{"type":155,"value":409},{"type":150,"tag":169,"props":17849,"children":17851},{"className":17850},[495],[17852],{"type":150,"tag":169,"props":17853,"children":17855},{"className":17854},[253,254],[17856,17885],{"type":150,"tag":169,"props":17857,"children":17859},{"className":17858},[259],[17860,17880],{"type":150,"tag":169,"props":17861,"children":17863},{"className":17862,"style":1369},[264],[17864],{"type":150,"tag":169,"props":17865,"children":17866},{"style":512},[17867,17871],{"type":150,"tag":169,"props":17868,"children":17870},{"className":17869,"style":517},[274],[],{"type":150,"tag":169,"props":17872,"children":17874},{"className":17873},[522,523,524,525],[17875],{"type":150,"tag":169,"props":17876,"children":17878},{"className":17877},[248,489,525],[17879],{"type":155,"value":1213},{"type":150,"tag":169,"props":17881,"children":17883},{"className":17882},[355],[17884],{"type":155,"value":358},{"type":150,"tag":169,"props":17886,"children":17888},{"className":17887},[259],[17889],{"type":150,"tag":169,"props":17890,"children":17892},{"className":17891,"style":544},[264],[17893],{"type":150,"tag":169,"props":17894,"children":17895},{},[],{"type":150,"tag":169,"props":17897,"children":17899},{"className":17898},[1004],[17900],{"type":155,"value":972},{"type":150,"tag":169,"props":17902,"children":17904},{"className":17903},[248],[17905,17910],{"type":150,"tag":169,"props":17906,"children":17908},{"className":17907,"style":3869},[248,489],[17909],{"type":155,"value":3443},{"type":150,"tag":169,"props":17911,"children":17913},{"className":17912},[495],[17914],{"type":150,"tag":169,"props":17915,"children":17917},{"className":17916},[253,254],[17918,17977],{"type":150,"tag":169,"props":17919,"children":17921},{"className":17920},[259],[17922,17972],{"type":150,"tag":169,"props":17923,"children":17925},{"className":17924,"style":1454},[264],[17926,17942],{"type":150,"tag":169,"props":17927,"children":17928},{"style":15747},[17929,17933],{"type":150,"tag":169,"props":17930,"children":17932},{"className":17931,"style":517},[274],[],{"type":150,"tag":169,"props":17934,"children":17936},{"className":17935},[522,523,524,525],[17937],{"type":150,"tag":169,"props":17938,"children":17940},{"className":17939,"style":2329},[248,489,525],[17941],{"type":155,"value":1845},{"type":150,"tag":169,"props":17943,"children":17944},{"style":1475},[17945,17949],{"type":150,"tag":169,"props":17946,"children":17948},{"className":17947,"style":517},[274],[],{"type":150,"tag":169,"props":17950,"children":17952},{"className":17951},[522,523,524,525],[17953],{"type":150,"tag":169,"props":17954,"children":17956},{"className":17955},[248,525],[17957,17962,17967],{"type":150,"tag":169,"props":17958,"children":17960},{"className":17959},[1004,525],[17961],{"type":155,"value":972},{"type":150,"tag":169,"props":17963,"children":17965},{"className":17964},[248,489,525],[17966],{"type":155,"value":977},{"type":150,"tag":169,"props":17968,"children":17970},{"className":17969},[1015,525],[17971],{"type":155,"value":982},{"type":150,"tag":169,"props":17973,"children":17975},{"className":17974},[355],[17976],{"type":155,"value":358},{"type":150,"tag":169,"props":17978,"children":17980},{"className":17979},[259],[17981],{"type":150,"tag":169,"props":17982,"children":17984},{"className":17983,"style":15804},[264],[17985],{"type":150,"tag":169,"props":17986,"children":17987},{},[],{"type":150,"tag":169,"props":17989,"children":17991},{"className":17990},[248],[17992,17997],{"type":150,"tag":169,"props":17993,"children":17995},{"className":17994,"style":3869},[248,489],[17996],{"type":155,"value":3443},{"type":150,"tag":169,"props":17998,"children":18000},{"className":17999},[495],[18001],{"type":150,"tag":169,"props":18002,"children":18004},{"className":18003},[253,254],[18005,18069],{"type":150,"tag":169,"props":18006,"children":18008},{"className":18007},[259],[18009,18064],{"type":150,"tag":169,"props":18010,"children":18012},{"className":18011,"style":1454},[264],[18013,18029],{"type":150,"tag":169,"props":18014,"children":18015},{"style":4286},[18016,18020],{"type":150,"tag":169,"props":18017,"children":18019},{"className":18018,"style":517},[274],[],{"type":150,"tag":169,"props":18021,"children":18023},{"className":18022},[522,523,524,525],[18024],{"type":150,"tag":169,"props":18025,"children":18027},{"className":18026,"style":2427},[248,489,525],[18028],{"type":155,"value":1876},{"type":150,"tag":169,"props":18030,"children":18031},{"style":1475},[18032,18036],{"type":150,"tag":169,"props":18033,"children":18035},{"className":18034,"style":517},[274],[],{"type":150,"tag":169,"props":18037,"children":18039},{"className":18038},[522,523,524,525],[18040],{"type":150,"tag":169,"props":18041,"children":18043},{"className":18042},[248,525],[18044,18049,18054,18059],{"type":150,"tag":169,"props":18045,"children":18047},{"className":18046},[1004,525],[18048],{"type":155,"value":972},{"type":150,"tag":169,"props":18050,"children":18052},{"className":18051},[248,489,525],[18053],{"type":155,"value":977},{"type":150,"tag":169,"props":18055,"children":18057},{"className":18056},[1015,525],[18058],{"type":155,"value":982},{"type":150,"tag":169,"props":18060,"children":18062},{"className":18061},[248,525],[18063],{"type":155,"value":2052},{"type":150,"tag":169,"props":18065,"children":18067},{"className":18066},[355],[18068],{"type":155,"value":358},{"type":150,"tag":169,"props":18070,"children":18072},{"className":18071},[259],[18073],{"type":150,"tag":169,"props":18074,"children":18076},{"className":18075,"style":4343},[264],[18077],{"type":150,"tag":169,"props":18078,"children":18079},{},[],{"type":150,"tag":169,"props":18081,"children":18083},{"className":18082},[1015],[18084],{"type":155,"value":982},{"type":155,"value":18086}," 算 Q，而是按 ",{"type":150,"tag":169,"props":18088,"children":18090},{"className":18089},[177],[18091,18184],{"type":150,"tag":169,"props":18092,"children":18094},{"className":18093},[182],[18095],{"type":150,"tag":185,"props":18096,"children":18097},{"xmlns":187},[18098],{"type":150,"tag":191,"props":18099,"children":18100},{},[18101,18179],{"type":150,"tag":195,"props":18102,"children":18103},{},[18104,18108,18119,18145,18149],{"type":150,"tag":417,"props":18105,"children":18106},{"stretchy":207},[18107],{"type":155,"value":972},{"type":150,"tag":401,"props":18109,"children":18110},{},[18111,18115],{"type":150,"tag":405,"props":18112,"children":18113},{},[18114],{"type":155,"value":409},{"type":150,"tag":405,"props":18116,"children":18117},{},[18118],{"type":155,"value":1213},{"type":150,"tag":1224,"props":18120,"children":18121},{},[18122,18126,18130],{"type":150,"tag":405,"props":18123,"children":18124},{},[18125],{"type":155,"value":3443},{"type":150,"tag":405,"props":18127,"children":18128},{},[18129],{"type":155,"value":1845},{"type":150,"tag":195,"props":18131,"children":18132},{},[18133,18137,18141],{"type":150,"tag":417,"props":18134,"children":18135},{"stretchy":207},[18136],{"type":155,"value":972},{"type":150,"tag":405,"props":18138,"children":18139},{},[18140],{"type":155,"value":977},{"type":150,"tag":417,"props":18142,"children":18143},{"stretchy":207},[18144],{"type":155,"value":982},{"type":150,"tag":417,"props":18146,"children":18147},{"stretchy":207},[18148],{"type":155,"value":982},{"type":150,"tag":1224,"props":18150,"children":18151},{},[18152,18156,18160],{"type":150,"tag":405,"props":18153,"children":18154},{},[18155],{"type":155,"value":3443},{"type":150,"tag":405,"props":18157,"children":18158},{},[18159],{"type":155,"value":1876},{"type":150,"tag":195,"props":18161,"children":18162},{},[18163,18167,18171,18175],{"type":150,"tag":417,"props":18164,"children":18165},{"stretchy":207},[18166],{"type":155,"value":972},{"type":150,"tag":405,"props":18168,"children":18169},{},[18170],{"type":155,"value":977},{"type":150,"tag":417,"props":18172,"children":18173},{"stretchy":207},[18174],{"type":155,"value":982},{"type":150,"tag":405,"props":18176,"children":18177},{"mathvariant":1796},[18178],{"type":155,"value":2052},{"type":150,"tag":223,"props":18180,"children":18181},{"encoding":225},[18182],{"type":155,"value":18183},"(x_t W_q^{(s)}) W_k^{(s)\\top}",{"type":150,"tag":169,"props":18185,"children":18187},{"className":18186,"ariaHidden":214},[232],[18188],{"type":150,"tag":169,"props":18189,"children":18191},{"className":18190},[237],[18192,18196,18201,18258,18345,18350],{"type":150,"tag":169,"props":18193,"children":18195},{"className":18194,"style":4200},[242],[],{"type":150,"tag":169,"props":18197,"children":18199},{"className":18198},[1004],[18200],{"type":155,"value":972},{"type":150,"tag":169,"props":18202,"children":18204},{"className":18203},[248],[18205,18210],{"type":150,"tag":169,"props":18206,"children":18208},{"className":18207},[248,489],[18209],{"type":155,"value":409},{"type":150,"tag":169,"props":18211,"children":18213},{"className":18212},[495],[18214],{"type":150,"tag":169,"props":18215,"children":18217},{"className":18216},[253,254],[18218,18247],{"type":150,"tag":169,"props":18219,"children":18221},{"className":18220},[259],[18222,18242],{"type":150,"tag":169,"props":18223,"children":18225},{"className":18224,"style":1369},[264],[18226],{"type":150,"tag":169,"props":18227,"children":18228},{"style":512},[18229,18233],{"type":150,"tag":169,"props":18230,"children":18232},{"className":18231,"style":517},[274],[],{"type":150,"tag":169,"props":18234,"children":18236},{"className":18235},[522,523,524,525],[18237],{"type":150,"tag":169,"props":18238,"children":18240},{"className":18239},[248,489,525],[18241],{"type":155,"value":1213},{"type":150,"tag":169,"props":18243,"children":18245},{"className":18244},[355],[18246],{"type":155,"value":358},{"type":150,"tag":169,"props":18248,"children":18250},{"className":18249},[259],[18251],{"type":150,"tag":169,"props":18252,"children":18254},{"className":18253,"style":544},[264],[18255],{"type":150,"tag":169,"props":18256,"children":18257},{},[],{"type":150,"tag":169,"props":18259,"children":18261},{"className":18260},[248],[18262,18267],{"type":150,"tag":169,"props":18263,"children":18265},{"className":18264,"style":3869},[248,489],[18266],{"type":155,"value":3443},{"type":150,"tag":169,"props":18268,"children":18270},{"className":18269},[495],[18271],{"type":150,"tag":169,"props":18272,"children":18274},{"className":18273},[253,254],[18275,18334],{"type":150,"tag":169,"props":18276,"children":18278},{"className":18277},[259],[18279,18329],{"type":150,"tag":169,"props":18280,"children":18282},{"className":18281,"style":1454},[264],[18283,18299],{"type":150,"tag":169,"props":18284,"children":18285},{"style":15747},[18286,18290],{"type":150,"tag":169,"props":18287,"children":18289},{"className":18288,"style":517},[274],[],{"type":150,"tag":169,"props":18291,"children":18293},{"className":18292},[522,523,524,525],[18294],{"type":150,"tag":169,"props":18295,"children":18297},{"className":18296,"style":2329},[248,489,525],[18298],{"type":155,"value":1845},{"type":150,"tag":169,"props":18300,"children":18301},{"style":1475},[18302,18306],{"type":150,"tag":169,"props":18303,"children":18305},{"className":18304,"style":517},[274],[],{"type":150,"tag":169,"props":18307,"children":18309},{"className":18308},[522,523,524,525],[18310],{"type":150,"tag":169,"props":18311,"children":18313},{"className":18312},[248,525],[18314,18319,18324],{"type":150,"tag":169,"props":18315,"children":18317},{"className":18316},[1004,525],[18318],{"type":155,"value":972},{"type":150,"tag":169,"props":18320,"children":18322},{"className":18321},[248,489,525],[18323],{"type":155,"value":977},{"type":150,"tag":169,"props":18325,"children":18327},{"className":18326},[1015,525],[18328],{"type":155,"value":982},{"type":150,"tag":169,"props":18330,"children":18332},{"className":18331},[355],[18333],{"type":155,"value":358},{"type":150,"tag":169,"props":18335,"children":18337},{"className":18336},[259],[18338],{"type":150,"tag":169,"props":18339,"children":18341},{"className":18340,"style":15804},[264],[18342],{"type":150,"tag":169,"props":18343,"children":18344},{},[],{"type":150,"tag":169,"props":18346,"children":18348},{"className":18347},[1015],[18349],{"type":155,"value":982},{"type":150,"tag":169,"props":18351,"children":18353},{"className":18352},[248],[18354,18359],{"type":150,"tag":169,"props":18355,"children":18357},{"className":18356,"style":3869},[248,489],[18358],{"type":155,"value":3443},{"type":150,"tag":169,"props":18360,"children":18362},{"className":18361},[495],[18363],{"type":150,"tag":169,"props":18364,"children":18366},{"className":18365},[253,254],[18367,18431],{"type":150,"tag":169,"props":18368,"children":18370},{"className":18369},[259],[18371,18426],{"type":150,"tag":169,"props":18372,"children":18374},{"className":18373,"style":1454},[264],[18375,18391],{"type":150,"tag":169,"props":18376,"children":18377},{"style":4286},[18378,18382],{"type":150,"tag":169,"props":18379,"children":18381},{"className":18380,"style":517},[274],[],{"type":150,"tag":169,"props":18383,"children":18385},{"className":18384},[522,523,524,525],[18386],{"type":150,"tag":169,"props":18387,"children":18389},{"className":18388,"style":2427},[248,489,525],[18390],{"type":155,"value":1876},{"type":150,"tag":169,"props":18392,"children":18393},{"style":1475},[18394,18398],{"type":150,"tag":169,"props":18395,"children":18397},{"className":18396,"style":517},[274],[],{"type":150,"tag":169,"props":18399,"children":18401},{"className":18400},[522,523,524,525],[18402],{"type":150,"tag":169,"props":18403,"children":18405},{"className":18404},[248,525],[18406,18411,18416,18421],{"type":150,"tag":169,"props":18407,"children":18409},{"className":18408},[1004,525],[18410],{"type":155,"value":972},{"type":150,"tag":169,"props":18412,"children":18414},{"className":18413},[248,489,525],[18415],{"type":155,"value":977},{"type":150,"tag":169,"props":18417,"children":18419},{"className":18418},[1015,525],[18420],{"type":155,"value":982},{"type":150,"tag":169,"props":18422,"children":18424},{"className":18423},[248,525],[18425],{"type":155,"value":2052},{"type":150,"tag":169,"props":18427,"children":18429},{"className":18428},[355],[18430],{"type":155,"value":358},{"type":150,"tag":169,"props":18432,"children":18434},{"className":18433},[259],[18435],{"type":150,"tag":169,"props":18436,"children":18438},{"className":18437,"style":4343},[264],[18439],{"type":150,"tag":169,"props":18440,"children":18441},{},[],{"type":155,"value":18443}," 算。虽然是串行的，但在低秩假设下计算量更少，理论精度损失也更小。",{"type":150,"tag":151,"props":18445,"children":18446},{},[18447,18449,18454,18456,18724],{"type":155,"value":18448},"不过别急。到这里 MLA 有个绕不开的缺陷：",{"type":150,"tag":158,"props":18450,"children":18451},{},[18452],{"type":155,"value":18453},"不兼容 RoPE",{"type":155,"value":18455},"。前面那步恒等变换能成立，关键在于 ",{"type":150,"tag":169,"props":18457,"children":18459},{"className":18458},[177],[18460,18533],{"type":150,"tag":169,"props":18461,"children":18463},{"className":18462},[182],[18464],{"type":150,"tag":185,"props":18465,"children":18466},{"xmlns":187},[18467],{"type":150,"tag":191,"props":18468,"children":18469},{},[18470,18529],{"type":150,"tag":195,"props":18471,"children":18472},{},[18473,18499],{"type":150,"tag":1224,"props":18474,"children":18475},{},[18476,18480,18484],{"type":150,"tag":405,"props":18477,"children":18478},{},[18479],{"type":155,"value":3443},{"type":150,"tag":405,"props":18481,"children":18482},{},[18483],{"type":155,"value":1845},{"type":150,"tag":195,"props":18485,"children":18486},{},[18487,18491,18495],{"type":150,"tag":417,"props":18488,"children":18489},{"stretchy":207},[18490],{"type":155,"value":972},{"type":150,"tag":405,"props":18492,"children":18493},{},[18494],{"type":155,"value":977},{"type":150,"tag":417,"props":18496,"children":18497},{"stretchy":207},[18498],{"type":155,"value":982},{"type":150,"tag":1224,"props":18500,"children":18501},{},[18502,18506,18510],{"type":150,"tag":405,"props":18503,"children":18504},{},[18505],{"type":155,"value":3443},{"type":150,"tag":405,"props":18507,"children":18508},{},[18509],{"type":155,"value":1876},{"type":150,"tag":195,"props":18511,"children":18512},{},[18513,18517,18521,18525],{"type":150,"tag":417,"props":18514,"children":18515},{"stretchy":207},[18516],{"type":155,"value":972},{"type":150,"tag":405,"props":18518,"children":18519},{},[18520],{"type":155,"value":977},{"type":150,"tag":417,"props":18522,"children":18523},{"stretchy":207},[18524],{"type":155,"value":982},{"type":150,"tag":405,"props":18526,"children":18527},{"mathvariant":1796},[18528],{"type":155,"value":2052},{"type":150,"tag":223,"props":18530,"children":18531},{"encoding":225},[18532],{"type":155,"value":15707},{"type":150,"tag":169,"props":18534,"children":18536},{"className":18535,"ariaHidden":214},[232],[18537],{"type":150,"tag":169,"props":18538,"children":18540},{"className":18539},[237],[18541,18545,18632],{"type":150,"tag":169,"props":18542,"children":18544},{"className":18543,"style":4200},[242],[],{"type":150,"tag":169,"props":18546,"children":18548},{"className":18547},[248],[18549,18554],{"type":150,"tag":169,"props":18550,"children":18552},{"className":18551,"style":3869},[248,489],[18553],{"type":155,"value":3443},{"type":150,"tag":169,"props":18555,"children":18557},{"className":18556},[495],[18558],{"type":150,"tag":169,"props":18559,"children":18561},{"className":18560},[253,254],[18562,18621],{"type":150,"tag":169,"props":18563,"children":18565},{"className":18564},[259],[18566,18616],{"type":150,"tag":169,"props":18567,"children":18569},{"className":18568,"style":1454},[264],[18570,18586],{"type":150,"tag":169,"props":18571,"children":18572},{"style":15747},[18573,18577],{"type":150,"tag":169,"props":18574,"children":18576},{"className":18575,"style":517},[274],[],{"type":150,"tag":169,"props":18578,"children":18580},{"className":18579},[522,523,524,525],[18581],{"type":150,"tag":169,"props":18582,"children":18584},{"className":18583,"style":2329},[248,489,525],[18585],{"type":155,"value":1845},{"type":150,"tag":169,"props":18587,"children":18588},{"style":1475},[18589,18593],{"type":150,"tag":169,"props":18590,"children":18592},{"className":18591,"style":517},[274],[],{"type":150,"tag":169,"props":18594,"children":18596},{"className":18595},[522,523,524,525],[18597],{"type":150,"tag":169,"props":18598,"children":18600},{"className":18599},[248,525],[18601,18606,18611],{"type":150,"tag":169,"props":18602,"children":18604},{"className":18603},[1004,525],[18605],{"type":155,"value":972},{"type":150,"tag":169,"props":18607,"children":18609},{"className":18608},[248,489,525],[18610],{"type":155,"value":977},{"type":150,"tag":169,"props":18612,"children":18614},{"className":18613},[1015,525],[18615],{"type":155,"value":982},{"type":150,"tag":169,"props":18617,"children":18619},{"className":18618},[355],[18620],{"type":155,"value":358},{"type":150,"tag":169,"props":18622,"children":18624},{"className":18623},[259],[18625],{"type":150,"tag":169,"props":18626,"children":18628},{"className":18627,"style":15804},[264],[18629],{"type":150,"tag":169,"props":18630,"children":18631},{},[],{"type":150,"tag":169,"props":18633,"children":18635},{"className":18634},[248],[18636,18641],{"type":150,"tag":169,"props":18637,"children":18639},{"className":18638,"style":3869},[248,489],[18640],{"type":155,"value":3443},{"type":150,"tag":169,"props":18642,"children":18644},{"className":18643},[495],[18645],{"type":150,"tag":169,"props":18646,"children":18648},{"className":18647},[253,254],[18649,18713],{"type":150,"tag":169,"props":18650,"children":18652},{"className":18651},[259],[18653,18708],{"type":150,"tag":169,"props":18654,"children":18656},{"className":18655,"style":1454},[264],[18657,18673],{"type":150,"tag":169,"props":18658,"children":18659},{"style":4286},[18660,18664],{"type":150,"tag":169,"props":18661,"children":18663},{"className":18662,"style":517},[274],[],{"type":150,"tag":169,"props":18665,"children":18667},{"className":18666},[522,523,524,525],[18668],{"type":150,"tag":169,"props":18669,"children":18671},{"className":18670,"style":2427},[248,489,525],[18672],{"type":155,"value":1876},{"type":150,"tag":169,"props":18674,"children":18675},{"style":1475},[18676,18680],{"type":150,"tag":169,"props":18677,"children":18679},{"className":18678,"style":517},[274],[],{"type":150,"tag":169,"props":18681,"children":18683},{"className":18682},[522,523,524,525],[18684],{"type":150,"tag":169,"props":18685,"children":18687},{"className":18686},[248,525],[18688,18693,18698,18703],{"type":150,"tag":169,"props":18689,"children":18691},{"className":18690},[1004,525],[18692],{"type":155,"value":972},{"type":150,"tag":169,"props":18694,"children":18696},{"className":18695},[248,489,525],[18697],{"type":155,"value":977},{"type":150,"tag":169,"props":18699,"children":18701},{"className":18700},[1015,525],[18702],{"type":155,"value":982},{"type":150,"tag":169,"props":18704,"children":18706},{"className":18705},[248,525],[18707],{"type":155,"value":2052},{"type":150,"tag":169,"props":18709,"children":18711},{"className":18710},[355],[18712],{"type":155,"value":358},{"type":150,"tag":169,"props":18714,"children":18716},{"className":18715},[259],[18717],{"type":150,"tag":169,"props":18718,"children":18720},{"className":18719,"style":4343},[264],[18721],{"type":150,"tag":169,"props":18722,"children":18723},{},[],{"type":155,"value":18725}," 是一个跟位置无关的固定矩阵。加上 RoPE 之后就不行了：",{"type":150,"tag":169,"props":18727,"children":18729},{"className":18728},[172],[18730],{"type":150,"tag":169,"props":18731,"children":18733},{"className":18732},[177],[18734,19048],{"type":150,"tag":169,"props":18735,"children":18737},{"className":18736},[182],[18738],{"type":150,"tag":185,"props":18739,"children":18740},{"xmlns":187,"display":188},[18741],{"type":150,"tag":191,"props":18742,"children":18743},{},[18744,19043],{"type":150,"tag":195,"props":18745,"children":18746},{},[18747,18773,18803,18807,18811,18822,18848,18859,18863,18867,18878,18904,18915,18926,18930,18941,18945,18971,18994,19024,19028],{"type":150,"tag":1224,"props":18748,"children":18749},{},[18750,18754,18758],{"type":150,"tag":405,"props":18751,"children":18752},{},[18753],{"type":155,"value":1845},{"type":150,"tag":405,"props":18755,"children":18756},{},[18757],{"type":155,"value":1213},{"type":150,"tag":195,"props":18759,"children":18760},{},[18761,18765,18769],{"type":150,"tag":417,"props":18762,"children":18763},{"stretchy":207},[18764],{"type":155,"value":972},{"type":150,"tag":405,"props":18766,"children":18767},{},[18768],{"type":155,"value":977},{"type":150,"tag":417,"props":18770,"children":18771},{"stretchy":207},[18772],{"type":155,"value":982},{"type":150,"tag":1224,"props":18774,"children":18775},{},[18776,18780,18784],{"type":150,"tag":405,"props":18777,"children":18778},{},[18779],{"type":155,"value":1876},{"type":150,"tag":405,"props":18781,"children":18782},{},[18783],{"type":155,"value":739},{"type":150,"tag":195,"props":18785,"children":18786},{},[18787,18791,18795,18799],{"type":150,"tag":417,"props":18788,"children":18789},{"stretchy":207},[18790],{"type":155,"value":972},{"type":150,"tag":405,"props":18792,"children":18793},{},[18794],{"type":155,"value":977},{"type":150,"tag":417,"props":18796,"children":18797},{"stretchy":207},[18798],{"type":155,"value":982},{"type":150,"tag":405,"props":18800,"children":18801},{"mathvariant":1796},[18802],{"type":155,"value":2052},{"type":150,"tag":417,"props":18804,"children":18805},{},[18806],{"type":155,"value":1045},{"type":150,"tag":417,"props":18808,"children":18809},{"stretchy":207},[18810],{"type":155,"value":972},{"type":150,"tag":401,"props":18812,"children":18813},{},[18814,18818],{"type":150,"tag":405,"props":18815,"children":18816},{},[18817],{"type":155,"value":409},{"type":150,"tag":405,"props":18819,"children":18820},{},[18821],{"type":155,"value":1213},{"type":150,"tag":1224,"props":18823,"children":18824},{},[18825,18829,18833],{"type":150,"tag":405,"props":18826,"children":18827},{},[18828],{"type":155,"value":3443},{"type":150,"tag":405,"props":18830,"children":18831},{},[18832],{"type":155,"value":1845},{"type":150,"tag":195,"props":18834,"children":18835},{},[18836,18840,18844],{"type":150,"tag":417,"props":18837,"children":18838},{"stretchy":207},[18839],{"type":155,"value":972},{"type":150,"tag":405,"props":18841,"children":18842},{},[18843],{"type":155,"value":977},{"type":150,"tag":417,"props":18845,"children":18846},{"stretchy":207},[18847],{"type":155,"value":982},{"type":150,"tag":401,"props":18849,"children":18850},{},[18851,18855],{"type":150,"tag":405,"props":18852,"children":18853},{},[18854],{"type":155,"value":754},{"type":150,"tag":405,"props":18856,"children":18857},{},[18858],{"type":155,"value":1213},{"type":150,"tag":417,"props":18860,"children":18861},{"stretchy":207},[18862],{"type":155,"value":982},{"type":150,"tag":417,"props":18864,"children":18865},{"stretchy":207},[18866],{"type":155,"value":972},{"type":150,"tag":401,"props":18868,"children":18869},{},[18870,18874],{"type":150,"tag":405,"props":18871,"children":18872},{},[18873],{"type":155,"value":10046},{"type":150,"tag":405,"props":18875,"children":18876},{},[18877],{"type":155,"value":739},{"type":150,"tag":1224,"props":18879,"children":18880},{},[18881,18885,18889],{"type":150,"tag":405,"props":18882,"children":18883},{},[18884],{"type":155,"value":3443},{"type":150,"tag":405,"props":18886,"children":18887},{},[18888],{"type":155,"value":1876},{"type":150,"tag":195,"props":18890,"children":18891},{},[18892,18896,18900],{"type":150,"tag":417,"props":18893,"children":18894},{"stretchy":207},[18895],{"type":155,"value":972},{"type":150,"tag":405,"props":18897,"children":18898},{},[18899],{"type":155,"value":977},{"type":150,"tag":417,"props":18901,"children":18902},{"stretchy":207},[18903],{"type":155,"value":982},{"type":150,"tag":401,"props":18905,"children":18906},{},[18907,18911],{"type":150,"tag":405,"props":18908,"children":18909},{},[18910],{"type":155,"value":754},{"type":150,"tag":405,"props":18912,"children":18913},{},[18914],{"type":155,"value":739},{"type":150,"tag":746,"props":18916,"children":18917},{},[18918,18922],{"type":150,"tag":417,"props":18919,"children":18920},{"stretchy":207},[18921],{"type":155,"value":982},{"type":150,"tag":405,"props":18923,"children":18924},{"mathvariant":1796},[18925],{"type":155,"value":2052},{"type":150,"tag":417,"props":18927,"children":18928},{},[18929],{"type":155,"value":1045},{"type":150,"tag":401,"props":18931,"children":18932},{},[18933,18937],{"type":150,"tag":405,"props":18934,"children":18935},{},[18936],{"type":155,"value":409},{"type":150,"tag":405,"props":18938,"children":18939},{},[18940],{"type":155,"value":1213},{"type":150,"tag":417,"props":18942,"children":18943},{"stretchy":207},[18944],{"type":155,"value":972},{"type":150,"tag":1224,"props":18946,"children":18947},{},[18948,18952,18956],{"type":150,"tag":405,"props":18949,"children":18950},{},[18951],{"type":155,"value":3443},{"type":150,"tag":405,"props":18953,"children":18954},{},[18955],{"type":155,"value":1845},{"type":150,"tag":195,"props":18957,"children":18958},{},[18959,18963,18967],{"type":150,"tag":417,"props":18960,"children":18961},{"stretchy":207},[18962],{"type":155,"value":972},{"type":150,"tag":405,"props":18964,"children":18965},{},[18966],{"type":155,"value":977},{"type":150,"tag":417,"props":18968,"children":18969},{"stretchy":207},[18970],{"type":155,"value":982},{"type":150,"tag":401,"props":18972,"children":18973},{},[18974,18978],{"type":150,"tag":405,"props":18975,"children":18976},{},[18977],{"type":155,"value":754},{"type":150,"tag":195,"props":18979,"children":18980},{},[18981,18985,18990],{"type":150,"tag":405,"props":18982,"children":18983},{},[18984],{"type":155,"value":1213},{"type":150,"tag":417,"props":18986,"children":18987},{},[18988],{"type":155,"value":18989},"−",{"type":150,"tag":405,"props":18991,"children":18992},{},[18993],{"type":155,"value":739},{"type":150,"tag":1224,"props":18995,"children":18996},{},[18997,19001,19005],{"type":150,"tag":405,"props":18998,"children":18999},{},[19000],{"type":155,"value":3443},{"type":150,"tag":405,"props":19002,"children":19003},{},[19004],{"type":155,"value":1876},{"type":150,"tag":195,"props":19006,"children":19007},{},[19008,19012,19016,19020],{"type":150,"tag":417,"props":19009,"children":19010},{"stretchy":207},[19011],{"type":155,"value":972},{"type":150,"tag":405,"props":19013,"children":19014},{},[19015],{"type":155,"value":977},{"type":150,"tag":417,"props":19017,"children":19018},{"stretchy":207},[19019],{"type":155,"value":982},{"type":150,"tag":405,"props":19021,"children":19022},{"mathvariant":1796},[19023],{"type":155,"value":2052},{"type":150,"tag":417,"props":19025,"children":19026},{"stretchy":207},[19027],{"type":155,"value":982},{"type":150,"tag":1224,"props":19029,"children":19030},{},[19031,19035,19039],{"type":150,"tag":405,"props":19032,"children":19033},{},[19034],{"type":155,"value":10046},{"type":150,"tag":405,"props":19036,"children":19037},{},[19038],{"type":155,"value":739},{"type":150,"tag":405,"props":19040,"children":19041},{"mathvariant":1796},[19042],{"type":155,"value":2052},{"type":150,"tag":223,"props":19044,"children":19045},{"encoding":225},[19046],{"type":155,"value":19047},"q_t^{(s)} k_i^{(s)\\top} = (x_t W_q^{(s)} R_t)(c_i W_k^{(s)} R_i)^\\top = x_t (W_q^{(s)} R_{t-i} W_k^{(s)\\top}) c_i^\\top",{"type":150,"tag":169,"props":19049,"children":19051},{"className":19050,"ariaHidden":214},[232],[19052,19252,19733],{"type":150,"tag":169,"props":19053,"children":19055},{"className":19054},[237],[19056,19060,19147,19239,19243,19248],{"type":150,"tag":169,"props":19057,"children":19059},{"className":19058,"style":3694},[242],[],{"type":150,"tag":169,"props":19061,"children":19063},{"className":19062},[248],[19064,19069],{"type":150,"tag":169,"props":19065,"children":19067},{"className":19066,"style":2329},[248,489],[19068],{"type":155,"value":1845},{"type":150,"tag":169,"props":19070,"children":19072},{"className":19071},[495],[19073],{"type":150,"tag":169,"props":19074,"children":19076},{"className":19075},[253,254],[19077,19136],{"type":150,"tag":169,"props":19078,"children":19080},{"className":19079},[259],[19081,19131],{"type":150,"tag":169,"props":19082,"children":19084},{"className":19083,"style":1454},[264],[19085,19101],{"type":150,"tag":169,"props":19086,"children":19087},{"style":2350},[19088,19092],{"type":150,"tag":169,"props":19089,"children":19091},{"className":19090,"style":517},[274],[],{"type":150,"tag":169,"props":19093,"children":19095},{"className":19094},[522,523,524,525],[19096],{"type":150,"tag":169,"props":19097,"children":19099},{"className":19098},[248,489,525],[19100],{"type":155,"value":1213},{"type":150,"tag":169,"props":19102,"children":19103},{"style":1475},[19104,19108],{"type":150,"tag":169,"props":19105,"children":19107},{"className":19106,"style":517},[274],[],{"type":150,"tag":169,"props":19109,"children":19111},{"className":19110},[522,523,524,525],[19112],{"type":150,"tag":169,"props":19113,"children":19115},{"className":19114},[248,525],[19116,19121,19126],{"type":150,"tag":169,"props":19117,"children":19119},{"className":19118},[1004,525],[19120],{"type":155,"value":972},{"type":150,"tag":169,"props":19122,"children":19124},{"className":19123},[248,489,525],[19125],{"type":155,"value":977},{"type":150,"tag":169,"props":19127,"children":19129},{"className":19128},[1015,525],[19130],{"type":155,"value":982},{"type":150,"tag":169,"props":19132,"children":19134},{"className":19133},[355],[19135],{"type":155,"value":358},{"type":150,"tag":169,"props":19137,"children":19139},{"className":19138},[259],[19140],{"type":150,"tag":169,"props":19141,"children":19143},{"className":19142,"style":1516},[264],[19144],{"type":150,"tag":169,"props":19145,"children":19146},{},[],{"type":150,"tag":169,"props":19148,"children":19150},{"className":19149},[248],[19151,19156],{"type":150,"tag":169,"props":19152,"children":19154},{"className":19153,"style":2427},[248,489],[19155],{"type":155,"value":1876},{"type":150,"tag":169,"props":19157,"children":19159},{"className":19158},[495],[19160],{"type":150,"tag":169,"props":19161,"children":19163},{"className":19162},[253,254],[19164,19228],{"type":150,"tag":169,"props":19165,"children":19167},{"className":19166},[259],[19168,19223],{"type":150,"tag":169,"props":19169,"children":19171},{"className":19170,"style":1454},[264],[19172,19188],{"type":150,"tag":169,"props":19173,"children":19174},{"style":2898},[19175,19179],{"type":150,"tag":169,"props":19176,"children":19178},{"className":19177,"style":517},[274],[],{"type":150,"tag":169,"props":19180,"children":19182},{"className":19181},[522,523,524,525],[19183],{"type":150,"tag":169,"props":19184,"children":19186},{"className":19185},[248,489,525],[19187],{"type":155,"value":739},{"type":150,"tag":169,"props":19189,"children":19190},{"style":1475},[19191,19195],{"type":150,"tag":169,"props":19192,"children":19194},{"className":19193,"style":517},[274],[],{"type":150,"tag":169,"props":19196,"children":19198},{"className":19197},[522,523,524,525],[19199],{"type":150,"tag":169,"props":19200,"children":19202},{"className":19201},[248,525],[19203,19208,19213,19218],{"type":150,"tag":169,"props":19204,"children":19206},{"className":19205},[1004,525],[19207],{"type":155,"value":972},{"type":150,"tag":169,"props":19209,"children":19211},{"className":19210},[248,489,525],[19212],{"type":155,"value":977},{"type":150,"tag":169,"props":19214,"children":19216},{"className":19215},[1015,525],[19217],{"type":155,"value":982},{"type":150,"tag":169,"props":19219,"children":19221},{"className":19220},[248,525],[19222],{"type":155,"value":2052},{"type":150,"tag":169,"props":19224,"children":19226},{"className":19225},[355],[19227],{"type":155,"value":358},{"type":150,"tag":169,"props":19229,"children":19231},{"className":19230},[259],[19232],{"type":150,"tag":169,"props":19233,"children":19235},{"className":19234,"style":2960},[264],[19236],{"type":150,"tag":169,"props":19237,"children":19238},{},[],{"type":150,"tag":169,"props":19240,"children":19242},{"className":19241,"style":839},[558],[],{"type":150,"tag":169,"props":19244,"children":19246},{"className":19245},[844],[19247],{"type":155,"value":1045},{"type":150,"tag":169,"props":19249,"children":19251},{"className":19250,"style":839},[558],[],{"type":150,"tag":169,"props":19253,"children":19255},{"className":19254},[237],[19256,19260,19265,19322,19409,19468,19473,19478,19535,19622,19679,19720,19724,19729],{"type":150,"tag":169,"props":19257,"children":19259},{"className":19258,"style":14938},[242],[],{"type":150,"tag":169,"props":19261,"children":19263},{"className":19262},[1004],[19264],{"type":155,"value":972},{"type":150,"tag":169,"props":19266,"children":19268},{"className":19267},[248],[19269,19274],{"type":150,"tag":169,"props":19270,"children":19272},{"className":19271},[248,489],[19273],{"type":155,"value":409},{"type":150,"tag":169,"props":19275,"children":19277},{"className":19276},[495],[19278],{"type":150,"tag":169,"props":19279,"children":19281},{"className":19280},[253,254],[19282,19311],{"type":150,"tag":169,"props":19283,"children":19285},{"className":19284},[259],[19286,19306],{"type":150,"tag":169,"props":19287,"children":19289},{"className":19288,"style":1369},[264],[19290],{"type":150,"tag":169,"props":19291,"children":19292},{"style":512},[19293,19297],{"type":150,"tag":169,"props":19294,"children":19296},{"className":19295,"style":517},[274],[],{"type":150,"tag":169,"props":19298,"children":19300},{"className":19299},[522,523,524,525],[19301],{"type":150,"tag":169,"props":19302,"children":19304},{"className":19303},[248,489,525],[19305],{"type":155,"value":1213},{"type":150,"tag":169,"props":19307,"children":19309},{"className":19308},[355],[19310],{"type":155,"value":358},{"type":150,"tag":169,"props":19312,"children":19314},{"className":19313},[259],[19315],{"type":150,"tag":169,"props":19316,"children":19318},{"className":19317,"style":544},[264],[19319],{"type":150,"tag":169,"props":19320,"children":19321},{},[],{"type":150,"tag":169,"props":19323,"children":19325},{"className":19324},[248],[19326,19331],{"type":150,"tag":169,"props":19327,"children":19329},{"className":19328,"style":3869},[248,489],[19330],{"type":155,"value":3443},{"type":150,"tag":169,"props":19332,"children":19334},{"className":19333},[495],[19335],{"type":150,"tag":169,"props":19336,"children":19338},{"className":19337},[253,254],[19339,19398],{"type":150,"tag":169,"props":19340,"children":19342},{"className":19341},[259],[19343,19393],{"type":150,"tag":169,"props":19344,"children":19346},{"className":19345,"style":3887},[264],[19347,19363],{"type":150,"tag":169,"props":19348,"children":19349},{"style":3891},[19350,19354],{"type":150,"tag":169,"props":19351,"children":19353},{"className":19352,"style":517},[274],[],{"type":150,"tag":169,"props":19355,"children":19357},{"className":19356},[522,523,524,525],[19358],{"type":150,"tag":169,"props":19359,"children":19361},{"className":19360,"style":2329},[248,489,525],[19362],{"type":155,"value":1845},{"type":150,"tag":169,"props":19364,"children":19365},{"style":3908},[19366,19370],{"type":150,"tag":169,"props":19367,"children":19369},{"className":19368,"style":517},[274],[],{"type":150,"tag":169,"props":19371,"children":19373},{"className":19372},[522,523,524,525],[19374],{"type":150,"tag":169,"props":19375,"children":19377},{"className":19376},[248,525],[19378,19383,19388],{"type":150,"tag":169,"props":19379,"children":19381},{"className":19380},[1004,525],[19382],{"type":155,"value":972},{"type":150,"tag":169,"props":19384,"children":19386},{"className":19385},[248,489,525],[19387],{"type":155,"value":977},{"type":150,"tag":169,"props":19389,"children":19391},{"className":19390},[1015,525],[19392],{"type":155,"value":982},{"type":150,"tag":169,"props":19394,"children":19396},{"className":19395},[355],[19397],{"type":155,"value":358},{"type":150,"tag":169,"props":19399,"children":19401},{"className":19400},[259],[19402],{"type":150,"tag":169,"props":19403,"children":19405},{"className":19404,"style":3949},[264],[19406],{"type":150,"tag":169,"props":19407,"children":19408},{},[],{"type":150,"tag":169,"props":19410,"children":19412},{"className":19411},[248],[19413,19419],{"type":150,"tag":169,"props":19414,"children":19417},{"className":19415,"style":19416},[248,489],"margin-right:0.0077em;",[19418],{"type":155,"value":754},{"type":150,"tag":169,"props":19420,"children":19422},{"className":19421},[495],[19423],{"type":150,"tag":169,"props":19424,"children":19426},{"className":19425},[253,254],[19427,19457],{"type":150,"tag":169,"props":19428,"children":19430},{"className":19429},[259],[19431,19452],{"type":150,"tag":169,"props":19432,"children":19434},{"className":19433,"style":1369},[264],[19435],{"type":150,"tag":169,"props":19436,"children":19438},{"style":19437},"top:-2.55em;margin-left:-0.0077em;margin-right:0.05em;",[19439,19443],{"type":150,"tag":169,"props":19440,"children":19442},{"className":19441,"style":517},[274],[],{"type":150,"tag":169,"props":19444,"children":19446},{"className":19445},[522,523,524,525],[19447],{"type":150,"tag":169,"props":19448,"children":19450},{"className":19449},[248,489,525],[19451],{"type":155,"value":1213},{"type":150,"tag":169,"props":19453,"children":19455},{"className":19454},[355],[19456],{"type":155,"value":358},{"type":150,"tag":169,"props":19458,"children":19460},{"className":19459},[259],[19461],{"type":150,"tag":169,"props":19462,"children":19464},{"className":19463,"style":544},[264],[19465],{"type":150,"tag":169,"props":19466,"children":19467},{},[],{"type":150,"tag":169,"props":19469,"children":19471},{"className":19470},[1015],[19472],{"type":155,"value":982},{"type":150,"tag":169,"props":19474,"children":19476},{"className":19475},[1004],[19477],{"type":155,"value":972},{"type":150,"tag":169,"props":19479,"children":19481},{"className":19480},[248],[19482,19487],{"type":150,"tag":169,"props":19483,"children":19485},{"className":19484},[248,489],[19486],{"type":155,"value":10046},{"type":150,"tag":169,"props":19488,"children":19490},{"className":19489},[495],[19491],{"type":150,"tag":169,"props":19492,"children":19494},{"className":19493},[253,254],[19495,19524],{"type":150,"tag":169,"props":19496,"children":19498},{"className":19497},[259],[19499,19519],{"type":150,"tag":169,"props":19500,"children":19502},{"className":19501,"style":802},[264],[19503],{"type":150,"tag":169,"props":19504,"children":19505},{"style":512},[19506,19510],{"type":150,"tag":169,"props":19507,"children":19509},{"className":19508,"style":517},[274],[],{"type":150,"tag":169,"props":19511,"children":19513},{"className":19512},[522,523,524,525],[19514],{"type":150,"tag":169,"props":19515,"children":19517},{"className":19516},[248,489,525],[19518],{"type":155,"value":739},{"type":150,"tag":169,"props":19520,"children":19522},{"className":19521},[355],[19523],{"type":155,"value":358},{"type":150,"tag":169,"props":19525,"children":19527},{"className":19526},[259],[19528],{"type":150,"tag":169,"props":19529,"children":19531},{"className":19530,"style":544},[264],[19532],{"type":150,"tag":169,"props":19533,"children":19534},{},[],{"type":150,"tag":169,"props":19536,"children":19538},{"className":19537},[248],[19539,19544],{"type":150,"tag":169,"props":19540,"children":19542},{"className":19541,"style":3869},[248,489],[19543],{"type":155,"value":3443},{"type":150,"tag":169,"props":19545,"children":19547},{"className":19546},[495],[19548],{"type":150,"tag":169,"props":19549,"children":19551},{"className":19550},[253,254],[19552,19611],{"type":150,"tag":169,"props":19553,"children":19555},{"className":19554},[259],[19556,19606],{"type":150,"tag":169,"props":19557,"children":19559},{"className":19558,"style":1454},[264],[19560,19576],{"type":150,"tag":169,"props":19561,"children":19562},{"style":4286},[19563,19567],{"type":150,"tag":169,"props":19564,"children":19566},{"className":19565,"style":517},[274],[],{"type":150,"tag":169,"props":19568,"children":19570},{"className":19569},[522,523,524,525],[19571],{"type":150,"tag":169,"props":19572,"children":19574},{"className":19573,"style":2427},[248,489,525],[19575],{"type":155,"value":1876},{"type":150,"tag":169,"props":19577,"children":19578},{"style":1475},[19579,19583],{"type":150,"tag":169,"props":19580,"children":19582},{"className":19581,"style":517},[274],[],{"type":150,"tag":169,"props":19584,"children":19586},{"className":19585},[522,523,524,525],[19587],{"type":150,"tag":169,"props":19588,"children":19590},{"className":19589},[248,525],[19591,19596,19601],{"type":150,"tag":169,"props":19592,"children":19594},{"className":19593},[1004,525],[19595],{"type":155,"value":972},{"type":150,"tag":169,"props":19597,"children":19599},{"className":19598},[248,489,525],[19600],{"type":155,"value":977},{"type":150,"tag":169,"props":19602,"children":19604},{"className":19603},[1015,525],[19605],{"type":155,"value":982},{"type":150,"tag":169,"props":19607,"children":19609},{"className":19608},[355],[19610],{"type":155,"value":358},{"type":150,"tag":169,"props":19612,"children":19614},{"className":19613},[259],[19615],{"type":150,"tag":169,"props":19616,"children":19618},{"className":19617,"style":4343},[264],[19619],{"type":150,"tag":169,"props":19620,"children":19621},{},[],{"type":150,"tag":169,"props":19623,"children":19625},{"className":19624},[248],[19626,19631],{"type":150,"tag":169,"props":19627,"children":19629},{"className":19628,"style":19416},[248,489],[19630],{"type":155,"value":754},{"type":150,"tag":169,"props":19632,"children":19634},{"className":19633},[495],[19635],{"type":150,"tag":169,"props":19636,"children":19638},{"className":19637},[253,254],[19639,19668],{"type":150,"tag":169,"props":19640,"children":19642},{"className":19641},[259],[19643,19663],{"type":150,"tag":169,"props":19644,"children":19646},{"className":19645,"style":802},[264],[19647],{"type":150,"tag":169,"props":19648,"children":19649},{"style":19437},[19650,19654],{"type":150,"tag":169,"props":19651,"children":19653},{"className":19652,"style":517},[274],[],{"type":150,"tag":169,"props":19655,"children":19657},{"className":19656},[522,523,524,525],[19658],{"type":150,"tag":169,"props":19659,"children":19661},{"className":19660},[248,489,525],[19662],{"type":155,"value":739},{"type":150,"tag":169,"props":19664,"children":19666},{"className":19665},[355],[19667],{"type":155,"value":358},{"type":150,"tag":169,"props":19669,"children":19671},{"className":19670},[259],[19672],{"type":150,"tag":169,"props":19673,"children":19675},{"className":19674,"style":544},[264],[19676],{"type":150,"tag":169,"props":19677,"children":19678},{},[],{"type":150,"tag":169,"props":19680,"children":19682},{"className":19681},[1015],[19683,19688],{"type":150,"tag":169,"props":19684,"children":19686},{"className":19685},[1015],[19687],{"type":155,"value":982},{"type":150,"tag":169,"props":19689,"children":19691},{"className":19690},[495],[19692],{"type":150,"tag":169,"props":19693,"children":19695},{"className":19694},[253],[19696],{"type":150,"tag":169,"props":19697,"children":19699},{"className":19698},[259],[19700],{"type":150,"tag":169,"props":19701,"children":19703},{"className":19702,"style":3999},[264],[19704],{"type":150,"tag":169,"props":19705,"children":19706},{"style":3908},[19707,19711],{"type":150,"tag":169,"props":19708,"children":19710},{"className":19709,"style":517},[274],[],{"type":150,"tag":169,"props":19712,"children":19714},{"className":19713},[522,523,524,525],[19715],{"type":150,"tag":169,"props":19716,"children":19718},{"className":19717},[248,525],[19719],{"type":155,"value":2052},{"type":150,"tag":169,"props":19721,"children":19723},{"className":19722,"style":839},[558],[],{"type":150,"tag":169,"props":19725,"children":19727},{"className":19726},[844],[19728],{"type":155,"value":1045},{"type":150,"tag":169,"props":19730,"children":19732},{"className":19731,"style":839},[558],[],{"type":150,"tag":169,"props":19734,"children":19736},{"className":19735},[237],[19737,19741,19798,19803,19890,19962,20054,20059],{"type":150,"tag":169,"props":19738,"children":19740},{"className":19739,"style":14938},[242],[],{"type":150,"tag":169,"props":19742,"children":19744},{"className":19743},[248],[19745,19750],{"type":150,"tag":169,"props":19746,"children":19748},{"className":19747},[248,489],[19749],{"type":155,"value":409},{"type":150,"tag":169,"props":19751,"children":19753},{"className":19752},[495],[19754],{"type":150,"tag":169,"props":19755,"children":19757},{"className":19756},[253,254],[19758,19787],{"type":150,"tag":169,"props":19759,"children":19761},{"className":19760},[259],[19762,19782],{"type":150,"tag":169,"props":19763,"children":19765},{"className":19764,"style":1369},[264],[19766],{"type":150,"tag":169,"props":19767,"children":19768},{"style":512},[19769,19773],{"type":150,"tag":169,"props":19770,"children":19772},{"className":19771,"style":517},[274],[],{"type":150,"tag":169,"props":19774,"children":19776},{"className":19775},[522,523,524,525],[19777],{"type":150,"tag":169,"props":19778,"children":19780},{"className":19779},[248,489,525],[19781],{"type":155,"value":1213},{"type":150,"tag":169,"props":19783,"children":19785},{"className":19784},[355],[19786],{"type":155,"value":358},{"type":150,"tag":169,"props":19788,"children":19790},{"className":19789},[259],[19791],{"type":150,"tag":169,"props":19792,"children":19794},{"className":19793,"style":544},[264],[19795],{"type":150,"tag":169,"props":19796,"children":19797},{},[],{"type":150,"tag":169,"props":19799,"children":19801},{"className":19800},[1004],[19802],{"type":155,"value":972},{"type":150,"tag":169,"props":19804,"children":19806},{"className":19805},[248],[19807,19812],{"type":150,"tag":169,"props":19808,"children":19810},{"className":19809,"style":3869},[248,489],[19811],{"type":155,"value":3443},{"type":150,"tag":169,"props":19813,"children":19815},{"className":19814},[495],[19816],{"type":150,"tag":169,"props":19817,"children":19819},{"className":19818},[253,254],[19820,19879],{"type":150,"tag":169,"props":19821,"children":19823},{"className":19822},[259],[19824,19874],{"type":150,"tag":169,"props":19825,"children":19827},{"className":19826,"style":3887},[264],[19828,19844],{"type":150,"tag":169,"props":19829,"children":19830},{"style":3891},[19831,19835],{"type":150,"tag":169,"props":19832,"children":19834},{"className":19833,"style":517},[274],[],{"type":150,"tag":169,"props":19836,"children":19838},{"className":19837},[522,523,524,525],[19839],{"type":150,"tag":169,"props":19840,"children":19842},{"className":19841,"style":2329},[248,489,525],[19843],{"type":155,"value":1845},{"type":150,"tag":169,"props":19845,"children":19846},{"style":3908},[19847,19851],{"type":150,"tag":169,"props":19848,"children":19850},{"className":19849,"style":517},[274],[],{"type":150,"tag":169,"props":19852,"children":19854},{"className":19853},[522,523,524,525],[19855],{"type":150,"tag":169,"props":19856,"children":19858},{"className":19857},[248,525],[19859,19864,19869],{"type":150,"tag":169,"props":19860,"children":19862},{"className":19861},[1004,525],[19863],{"type":155,"value":972},{"type":150,"tag":169,"props":19865,"children":19867},{"className":19866},[248,489,525],[19868],{"type":155,"value":977},{"type":150,"tag":169,"props":19870,"children":19872},{"className":19871},[1015,525],[19873],{"type":155,"value":982},{"type":150,"tag":169,"props":19875,"children":19877},{"className":19876},[355],[19878],{"type":155,"value":358},{"type":150,"tag":169,"props":19880,"children":19882},{"className":19881},[259],[19883],{"type":150,"tag":169,"props":19884,"children":19886},{"className":19885,"style":3949},[264],[19887],{"type":150,"tag":169,"props":19888,"children":19889},{},[],{"type":150,"tag":169,"props":19891,"children":19893},{"className":19892},[248],[19894,19899],{"type":150,"tag":169,"props":19895,"children":19897},{"className":19896,"style":19416},[248,489],[19898],{"type":155,"value":754},{"type":150,"tag":169,"props":19900,"children":19902},{"className":19901},[495],[19903],{"type":150,"tag":169,"props":19904,"children":19906},{"className":19905},[253,254],[19907,19950],{"type":150,"tag":169,"props":19908,"children":19910},{"className":19909},[259],[19911,19945],{"type":150,"tag":169,"props":19912,"children":19914},{"className":19913,"style":802},[264],[19915],{"type":150,"tag":169,"props":19916,"children":19917},{"style":19437},[19918,19922],{"type":150,"tag":169,"props":19919,"children":19921},{"className":19920,"style":517},[274],[],{"type":150,"tag":169,"props":19923,"children":19925},{"className":19924},[522,523,524,525],[19926],{"type":150,"tag":169,"props":19927,"children":19929},{"className":19928},[248,525],[19930,19935,19940],{"type":150,"tag":169,"props":19931,"children":19933},{"className":19932},[248,489,525],[19934],{"type":155,"value":1213},{"type":150,"tag":169,"props":19936,"children":19938},{"className":19937},[5172,525],[19939],{"type":155,"value":18989},{"type":150,"tag":169,"props":19941,"children":19943},{"className":19942},[248,489,525],[19944],{"type":155,"value":739},{"type":150,"tag":169,"props":19946,"children":19948},{"className":19947},[355],[19949],{"type":155,"value":358},{"type":150,"tag":169,"props":19951,"children":19953},{"className":19952},[259],[19954],{"type":150,"tag":169,"props":19955,"children":19958},{"className":19956,"style":19957},[264],"height:0.2083em;",[19959],{"type":150,"tag":169,"props":19960,"children":19961},{},[],{"type":150,"tag":169,"props":19963,"children":19965},{"className":19964},[248],[19966,19971],{"type":150,"tag":169,"props":19967,"children":19969},{"className":19968,"style":3869},[248,489],[19970],{"type":155,"value":3443},{"type":150,"tag":169,"props":19972,"children":19974},{"className":19973},[495],[19975],{"type":150,"tag":169,"props":19976,"children":19978},{"className":19977},[253,254],[19979,20043],{"type":150,"tag":169,"props":19980,"children":19982},{"className":19981},[259],[19983,20038],{"type":150,"tag":169,"props":19984,"children":19986},{"className":19985,"style":1454},[264],[19987,20003],{"type":150,"tag":169,"props":19988,"children":19989},{"style":4286},[19990,19994],{"type":150,"tag":169,"props":19991,"children":19993},{"className":19992,"style":517},[274],[],{"type":150,"tag":169,"props":19995,"children":19997},{"className":19996},[522,523,524,525],[19998],{"type":150,"tag":169,"props":19999,"children":20001},{"className":20000,"style":2427},[248,489,525],[20002],{"type":155,"value":1876},{"type":150,"tag":169,"props":20004,"children":20005},{"style":1475},[20006,20010],{"type":150,"tag":169,"props":20007,"children":20009},{"className":20008,"style":517},[274],[],{"type":150,"tag":169,"props":20011,"children":20013},{"className":20012},[522,523,524,525],[20014],{"type":150,"tag":169,"props":20015,"children":20017},{"className":20016},[248,525],[20018,20023,20028,20033],{"type":150,"tag":169,"props":20019,"children":20021},{"className":20020},[1004,525],[20022],{"type":155,"value":972},{"type":150,"tag":169,"props":20024,"children":20026},{"className":20025},[248,489,525],[20027],{"type":155,"value":977},{"type":150,"tag":169,"props":20029,"children":20031},{"className":20030},[1015,525],[20032],{"type":155,"value":982},{"type":150,"tag":169,"props":20034,"children":20036},{"className":20035},[248,525],[20037],{"type":155,"value":2052},{"type":150,"tag":169,"props":20039,"children":20041},{"className":20040},[355],[20042],{"type":155,"value":358},{"type":150,"tag":169,"props":20044,"children":20046},{"className":20045},[259],[20047],{"type":150,"tag":169,"props":20048,"children":20050},{"className":20049,"style":4343},[264],[20051],{"type":150,"tag":169,"props":20052,"children":20053},{},[],{"type":150,"tag":169,"props":20055,"children":20057},{"className":20056},[1015],[20058],{"type":155,"value":982},{"type":150,"tag":169,"props":20060,"children":20062},{"className":20061},[248],[20063,20068],{"type":150,"tag":169,"props":20064,"children":20066},{"className":20065},[248,489],[20067],{"type":155,"value":10046},{"type":150,"tag":169,"props":20069,"children":20071},{"className":20070},[495],[20072],{"type":150,"tag":169,"props":20073,"children":20075},{"className":20074},[253,254],[20076,20121],{"type":150,"tag":169,"props":20077,"children":20079},{"className":20078},[259],[20080,20116],{"type":150,"tag":169,"props":20081,"children":20083},{"className":20082,"style":3999},[264],[20084,20100],{"type":150,"tag":169,"props":20085,"children":20086},{"style":15578},[20087,20091],{"type":150,"tag":169,"props":20088,"children":20090},{"className":20089,"style":517},[274],[],{"type":150,"tag":169,"props":20092,"children":20094},{"className":20093},[522,523,524,525],[20095],{"type":150,"tag":169,"props":20096,"children":20098},{"className":20097},[248,489,525],[20099],{"type":155,"value":739},{"type":150,"tag":169,"props":20101,"children":20102},{"style":3908},[20103,20107],{"type":150,"tag":169,"props":20104,"children":20106},{"className":20105,"style":517},[274],[],{"type":150,"tag":169,"props":20108,"children":20110},{"className":20109},[522,523,524,525],[20111],{"type":150,"tag":169,"props":20112,"children":20114},{"className":20113},[248,525],[20115],{"type":155,"value":2052},{"type":150,"tag":169,"props":20117,"children":20119},{"className":20118},[355],[20120],{"type":155,"value":358},{"type":150,"tag":169,"props":20122,"children":20124},{"className":20123},[259],[20125],{"type":150,"tag":169,"props":20126,"children":20128},{"className":20127,"style":4728},[264],[20129],{"type":150,"tag":169,"props":20130,"children":20131},{},[],{"type":150,"tag":151,"props":20133,"children":20134},{},[20135,20497,20499,20577,20579,20682],{"type":150,"tag":169,"props":20136,"children":20138},{"className":20137},[177],[20139,20235],{"type":150,"tag":169,"props":20140,"children":20142},{"className":20141},[182],[20143],{"type":150,"tag":185,"props":20144,"children":20145},{"xmlns":187},[20146],{"type":150,"tag":191,"props":20147,"children":20148},{},[20149,20230],{"type":150,"tag":195,"props":20150,"children":20151},{},[20152,20178,20200],{"type":150,"tag":1224,"props":20153,"children":20154},{},[20155,20159,20163],{"type":150,"tag":405,"props":20156,"children":20157},{},[20158],{"type":155,"value":3443},{"type":150,"tag":405,"props":20160,"children":20161},{},[20162],{"type":155,"value":1845},{"type":150,"tag":195,"props":20164,"children":20165},{},[20166,20170,20174],{"type":150,"tag":417,"props":20167,"children":20168},{"stretchy":207},[20169],{"type":155,"value":972},{"type":150,"tag":405,"props":20171,"children":20172},{},[20173],{"type":155,"value":977},{"type":150,"tag":417,"props":20175,"children":20176},{"stretchy":207},[20177],{"type":155,"value":982},{"type":150,"tag":401,"props":20179,"children":20180},{},[20181,20185],{"type":150,"tag":405,"props":20182,"children":20183},{},[20184],{"type":155,"value":754},{"type":150,"tag":195,"props":20186,"children":20187},{},[20188,20192,20196],{"type":150,"tag":405,"props":20189,"children":20190},{},[20191],{"type":155,"value":1213},{"type":150,"tag":417,"props":20193,"children":20194},{},[20195],{"type":155,"value":18989},{"type":150,"tag":405,"props":20197,"children":20198},{},[20199],{"type":155,"value":739},{"type":150,"tag":1224,"props":20201,"children":20202},{},[20203,20207,20211],{"type":150,"tag":405,"props":20204,"children":20205},{},[20206],{"type":155,"value":3443},{"type":150,"tag":405,"props":20208,"children":20209},{},[20210],{"type":155,"value":1876},{"type":150,"tag":195,"props":20212,"children":20213},{},[20214,20218,20222,20226],{"type":150,"tag":417,"props":20215,"children":20216},{"stretchy":207},[20217],{"type":155,"value":972},{"type":150,"tag":405,"props":20219,"children":20220},{},[20221],{"type":155,"value":977},{"type":150,"tag":417,"props":20223,"children":20224},{"stretchy":207},[20225],{"type":155,"value":982},{"type":150,"tag":405,"props":20227,"children":20228},{"mathvariant":1796},[20229],{"type":155,"value":2052},{"type":150,"tag":223,"props":20231,"children":20232},{"encoding":225},[20233],{"type":155,"value":20234},"W_q^{(s)} R_{t-i} W_k^{(s)\\top}",{"type":150,"tag":169,"props":20236,"children":20238},{"className":20237,"ariaHidden":214},[232],[20239],{"type":150,"tag":169,"props":20240,"children":20242},{"className":20241},[237],[20243,20247,20334,20405],{"type":150,"tag":169,"props":20244,"children":20246},{"className":20245,"style":4200},[242],[],{"type":150,"tag":169,"props":20248,"children":20250},{"className":20249},[248],[20251,20256],{"type":150,"tag":169,"props":20252,"children":20254},{"className":20253,"style":3869},[248,489],[20255],{"type":155,"value":3443},{"type":150,"tag":169,"props":20257,"children":20259},{"className":20258},[495],[20260],{"type":150,"tag":169,"props":20261,"children":20263},{"className":20262},[253,254],[20264,20323],{"type":150,"tag":169,"props":20265,"children":20267},{"className":20266},[259],[20268,20318],{"type":150,"tag":169,"props":20269,"children":20271},{"className":20270,"style":1454},[264],[20272,20288],{"type":150,"tag":169,"props":20273,"children":20274},{"style":15747},[20275,20279],{"type":150,"tag":169,"props":20276,"children":20278},{"className":20277,"style":517},[274],[],{"type":150,"tag":169,"props":20280,"children":20282},{"className":20281},[522,523,524,525],[20283],{"type":150,"tag":169,"props":20284,"children":20286},{"className":20285,"style":2329},[248,489,525],[20287],{"type":155,"value":1845},{"type":150,"tag":169,"props":20289,"children":20290},{"style":1475},[20291,20295],{"type":150,"tag":169,"props":20292,"children":20294},{"className":20293,"style":517},[274],[],{"type":150,"tag":169,"props":20296,"children":20298},{"className":20297},[522,523,524,525],[20299],{"type":150,"tag":169,"props":20300,"children":20302},{"className":20301},[248,525],[20303,20308,20313],{"type":150,"tag":169,"props":20304,"children":20306},{"className":20305},[1004,525],[20307],{"type":155,"value":972},{"type":150,"tag":169,"props":20309,"children":20311},{"className":20310},[248,489,525],[20312],{"type":155,"value":977},{"type":150,"tag":169,"props":20314,"children":20316},{"className":20315},[1015,525],[20317],{"type":155,"value":982},{"type":150,"tag":169,"props":20319,"children":20321},{"className":20320},[355],[20322],{"type":155,"value":358},{"type":150,"tag":169,"props":20324,"children":20326},{"className":20325},[259],[20327],{"type":150,"tag":169,"props":20328,"children":20330},{"className":20329,"style":15804},[264],[20331],{"type":150,"tag":169,"props":20332,"children":20333},{},[],{"type":150,"tag":169,"props":20335,"children":20337},{"className":20336},[248],[20338,20343],{"type":150,"tag":169,"props":20339,"children":20341},{"className":20340,"style":19416},[248,489],[20342],{"type":155,"value":754},{"type":150,"tag":169,"props":20344,"children":20346},{"className":20345},[495],[20347],{"type":150,"tag":169,"props":20348,"children":20350},{"className":20349},[253,254],[20351,20394],{"type":150,"tag":169,"props":20352,"children":20354},{"className":20353},[259],[20355,20389],{"type":150,"tag":169,"props":20356,"children":20358},{"className":20357,"style":802},[264],[20359],{"type":150,"tag":169,"props":20360,"children":20361},{"style":19437},[20362,20366],{"type":150,"tag":169,"props":20363,"children":20365},{"className":20364,"style":517},[274],[],{"type":150,"tag":169,"props":20367,"children":20369},{"className":20368},[522,523,524,525],[20370],{"type":150,"tag":169,"props":20371,"children":20373},{"className":20372},[248,525],[20374,20379,20384],{"type":150,"tag":169,"props":20375,"children":20377},{"className":20376},[248,489,525],[20378],{"type":155,"value":1213},{"type":150,"tag":169,"props":20380,"children":20382},{"className":20381},[5172,525],[20383],{"type":155,"value":18989},{"type":150,"tag":169,"props":20385,"children":20387},{"className":20386},[248,489,525],[20388],{"type":155,"value":739},{"type":150,"tag":169,"props":20390,"children":20392},{"className":20391},[355],[20393],{"type":155,"value":358},{"type":150,"tag":169,"props":20395,"children":20397},{"className":20396},[259],[20398],{"type":150,"tag":169,"props":20399,"children":20401},{"className":20400,"style":19957},[264],[20402],{"type":150,"tag":169,"props":20403,"children":20404},{},[],{"type":150,"tag":169,"props":20406,"children":20408},{"className":20407},[248],[20409,20414],{"type":150,"tag":169,"props":20410,"children":20412},{"className":20411,"style":3869},[248,489],[20413],{"type":155,"value":3443},{"type":150,"tag":169,"props":20415,"children":20417},{"className":20416},[495],[20418],{"type":150,"tag":169,"props":20419,"children":20421},{"className":20420},[253,254],[20422,20486],{"type":150,"tag":169,"props":20423,"children":20425},{"className":20424},[259],[20426,20481],{"type":150,"tag":169,"props":20427,"children":20429},{"className":20428,"style":1454},[264],[20430,20446],{"type":150,"tag":169,"props":20431,"children":20432},{"style":4286},[20433,20437],{"type":150,"tag":169,"props":20434,"children":20436},{"className":20435,"style":517},[274],[],{"type":150,"tag":169,"props":20438,"children":20440},{"className":20439},[522,523,524,525],[20441],{"type":150,"tag":169,"props":20442,"children":20444},{"className":20443,"style":2427},[248,489,525],[20445],{"type":155,"value":1876},{"type":150,"tag":169,"props":20447,"children":20448},{"style":1475},[20449,20453],{"type":150,"tag":169,"props":20450,"children":20452},{"className":20451,"style":517},[274],[],{"type":150,"tag":169,"props":20454,"children":20456},{"className":20455},[522,523,524,525],[20457],{"type":150,"tag":169,"props":20458,"children":20460},{"className":20459},[248,525],[20461,20466,20471,20476],{"type":150,"tag":169,"props":20462,"children":20464},{"className":20463},[1004,525],[20465],{"type":155,"value":972},{"type":150,"tag":169,"props":20467,"children":20469},{"className":20468},[248,489,525],[20470],{"type":155,"value":977},{"type":150,"tag":169,"props":20472,"children":20474},{"className":20473},[1015,525],[20475],{"type":155,"value":982},{"type":150,"tag":169,"props":20477,"children":20479},{"className":20478},[248,525],[20480],{"type":155,"value":2052},{"type":150,"tag":169,"props":20482,"children":20484},{"className":20483},[355],[20485],{"type":155,"value":358},{"type":150,"tag":169,"props":20487,"children":20489},{"className":20488},[259],[20490],{"type":150,"tag":169,"props":20491,"children":20493},{"className":20492,"style":4343},[264],[20494],{"type":150,"tag":169,"props":20495,"children":20496},{},[],{"type":155,"value":20498}," 跟位置差 ",{"type":150,"tag":169,"props":20500,"children":20502},{"className":20501},[177],[20503,20533],{"type":150,"tag":169,"props":20504,"children":20506},{"className":20505},[182],[20507],{"type":150,"tag":185,"props":20508,"children":20509},{"xmlns":187},[20510],{"type":150,"tag":191,"props":20511,"children":20512},{},[20513,20528],{"type":150,"tag":195,"props":20514,"children":20515},{},[20516,20520,20524],{"type":150,"tag":405,"props":20517,"children":20518},{},[20519],{"type":155,"value":1213},{"type":150,"tag":417,"props":20521,"children":20522},{},[20523],{"type":155,"value":18989},{"type":150,"tag":405,"props":20525,"children":20526},{},[20527],{"type":155,"value":739},{"type":150,"tag":223,"props":20529,"children":20530},{"encoding":225},[20531],{"type":155,"value":20532},"t - i",{"type":150,"tag":169,"props":20534,"children":20536},{"className":20535,"ariaHidden":214},[232],[20537,20563],{"type":150,"tag":169,"props":20538,"children":20540},{"className":20539},[237],[20541,20545,20550,20554,20559],{"type":150,"tag":169,"props":20542,"children":20544},{"className":20543,"style":5157},[242],[],{"type":150,"tag":169,"props":20546,"children":20548},{"className":20547},[248,489],[20549],{"type":155,"value":1213},{"type":150,"tag":169,"props":20551,"children":20553},{"className":20552,"style":5167},[558],[],{"type":150,"tag":169,"props":20555,"children":20557},{"className":20556},[5172],[20558],{"type":155,"value":18989},{"type":150,"tag":169,"props":20560,"children":20562},{"className":20561,"style":5167},[558],[],{"type":150,"tag":169,"props":20564,"children":20566},{"className":20565},[237],[20567,20572],{"type":150,"tag":169,"props":20568,"children":20571},{"className":20569,"style":20570},[242],"height:0.6595em;",[],{"type":150,"tag":169,"props":20573,"children":20575},{"className":20574},[248,489],[20576],{"type":155,"value":739},{"type":155,"value":20578}," 相关，无法合并成一个固定矩阵。放弃 RoPE 换 ALIBI 之类的 attention bias 是一种办法，但实验显示明显不如 RoPE。最后 MLA 采取的是混合方案：每个 head 的 Q、K 新增 ",{"type":150,"tag":169,"props":20580,"children":20582},{"className":20581},[177],[20583,20613],{"type":150,"tag":169,"props":20584,"children":20586},{"className":20585},[182],[20587],{"type":150,"tag":185,"props":20588,"children":20589},{"xmlns":187},[20590],{"type":150,"tag":191,"props":20591,"children":20592},{},[20593,20608],{"type":150,"tag":195,"props":20594,"children":20595},{},[20596],{"type":150,"tag":401,"props":20597,"children":20598},{},[20599,20603],{"type":150,"tag":405,"props":20600,"children":20601},{},[20602],{"type":155,"value":759},{"type":150,"tag":405,"props":20604,"children":20605},{},[20606],{"type":155,"value":20607},"r",{"type":150,"tag":223,"props":20609,"children":20610},{"encoding":225},[20611],{"type":155,"value":20612},"d_r",{"type":150,"tag":169,"props":20614,"children":20616},{"className":20615,"ariaHidden":214},[232],[20617],{"type":150,"tag":169,"props":20618,"children":20620},{"className":20619},[237],[20621,20625],{"type":150,"tag":169,"props":20622,"children":20624},{"className":20623,"style":4935},[242],[],{"type":150,"tag":169,"props":20626,"children":20628},{"className":20627},[248],[20629,20634],{"type":150,"tag":169,"props":20630,"children":20632},{"className":20631},[248,489],[20633],{"type":155,"value":759},{"type":150,"tag":169,"props":20635,"children":20637},{"className":20636},[495],[20638],{"type":150,"tag":169,"props":20639,"children":20641},{"className":20640},[253,254],[20642,20671],{"type":150,"tag":169,"props":20643,"children":20645},{"className":20644},[259],[20646,20666],{"type":150,"tag":169,"props":20647,"children":20649},{"className":20648,"style":5039},[264],[20650],{"type":150,"tag":169,"props":20651,"children":20652},{"style":512},[20653,20657],{"type":150,"tag":169,"props":20654,"children":20656},{"className":20655,"style":517},[274],[],{"type":150,"tag":169,"props":20658,"children":20660},{"className":20659},[522,523,524,525],[20661],{"type":150,"tag":169,"props":20662,"children":20664},{"className":20663,"style":6968},[248,489,525],[20665],{"type":155,"value":20607},{"type":150,"tag":169,"props":20667,"children":20669},{"className":20668},[355],[20670],{"type":155,"value":358},{"type":150,"tag":169,"props":20672,"children":20674},{"className":20673},[259],[20675],{"type":150,"tag":169,"props":20676,"children":20678},{"className":20677,"style":544},[264],[20679],{"type":150,"tag":169,"props":20680,"children":20681},{},[],{"type":155,"value":20683}," 个维度专门用来加 RoPE，其中 K 新增的这部分在 head 间共享。",{"type":150,"tag":151,"props":20685,"children":20686},{},[20687,20689,20790,20792,20943,20945,21082],{"type":155,"value":20688},"这样没有 RoPE 的维度继续走前面的压缩，带 RoPE 的维度补充位置信息。",{"type":150,"tag":169,"props":20690,"children":20692},{"className":20691},[177],[20693,20721],{"type":150,"tag":169,"props":20694,"children":20696},{"className":20695},[182],[20697],{"type":150,"tag":185,"props":20698,"children":20699},{"xmlns":187},[20700],{"type":150,"tag":191,"props":20701,"children":20702},{},[20703,20717],{"type":150,"tag":195,"props":20704,"children":20705},{},[20706],{"type":150,"tag":401,"props":20707,"children":20708},{},[20709,20713],{"type":150,"tag":405,"props":20710,"children":20711},{},[20712],{"type":155,"value":759},{"type":150,"tag":405,"props":20714,"children":20715},{},[20716],{"type":155,"value":20607},{"type":150,"tag":223,"props":20718,"children":20719},{"encoding":225},[20720],{"type":155,"value":20612},{"type":150,"tag":169,"props":20722,"children":20724},{"className":20723,"ariaHidden":214},[232],[20725],{"type":150,"tag":169,"props":20726,"children":20728},{"className":20727},[237],[20729,20733],{"type":150,"tag":169,"props":20730,"children":20732},{"className":20731,"style":4935},[242],[],{"type":150,"tag":169,"props":20734,"children":20736},{"className":20735},[248],[20737,20742],{"type":150,"tag":169,"props":20738,"children":20740},{"className":20739},[248,489],[20741],{"type":155,"value":759},{"type":150,"tag":169,"props":20743,"children":20745},{"className":20744},[495],[20746],{"type":150,"tag":169,"props":20747,"children":20749},{"className":20748},[253,254],[20750,20779],{"type":150,"tag":169,"props":20751,"children":20753},{"className":20752},[259],[20754,20774],{"type":150,"tag":169,"props":20755,"children":20757},{"className":20756,"style":5039},[264],[20758],{"type":150,"tag":169,"props":20759,"children":20760},{"style":512},[20761,20765],{"type":150,"tag":169,"props":20762,"children":20764},{"className":20763,"style":517},[274],[],{"type":150,"tag":169,"props":20766,"children":20768},{"className":20767},[522,523,524,525],[20769],{"type":150,"tag":169,"props":20770,"children":20772},{"className":20771,"style":6968},[248,489,525],[20773],{"type":155,"value":20607},{"type":150,"tag":169,"props":20775,"children":20777},{"className":20776},[355],[20778],{"type":155,"value":358},{"type":150,"tag":169,"props":20780,"children":20782},{"className":20781},[259],[20783],{"type":150,"tag":169,"props":20784,"children":20786},{"className":20785,"style":544},[264],[20787],{"type":150,"tag":169,"props":20788,"children":20789},{},[],{"type":155,"value":20791}," 取 ",{"type":150,"tag":169,"props":20793,"children":20795},{"className":20794},[177],[20796,20842],{"type":150,"tag":169,"props":20797,"children":20799},{"className":20798},[182],[20800],{"type":150,"tag":185,"props":20801,"children":20802},{"xmlns":187},[20803],{"type":150,"tag":191,"props":20804,"children":20805},{},[20806,20837],{"type":150,"tag":195,"props":20807,"children":20808},{},[20809,20820,20824,20828,20832],{"type":150,"tag":401,"props":20810,"children":20811},{},[20812,20816],{"type":150,"tag":405,"props":20813,"children":20814},{},[20815],{"type":155,"value":759},{"type":150,"tag":405,"props":20817,"children":20818},{},[20819],{"type":155,"value":1876},{"type":150,"tag":405,"props":20821,"children":20822},{"mathvariant":1796},[20823],{"type":155,"value":4914},{"type":150,"tag":411,"props":20825,"children":20826},{},[20827],{"type":155,"value":433},{"type":150,"tag":417,"props":20829,"children":20830},{},[20831],{"type":155,"value":1045},{"type":150,"tag":411,"props":20833,"children":20834},{},[20835],{"type":155,"value":20836},"64",{"type":150,"tag":223,"props":20838,"children":20839},{"encoding":225},[20840],{"type":155,"value":20841},"d_k\u002F2 = 64",{"type":150,"tag":169,"props":20843,"children":20845},{"className":20844,"ariaHidden":214},[232],[20846,20930],{"type":150,"tag":169,"props":20847,"children":20849},{"className":20848},[237],[20850,20854,20911,20917,20921,20926],{"type":150,"tag":169,"props":20851,"children":20853},{"className":20852,"style":999},[242],[],{"type":150,"tag":169,"props":20855,"children":20857},{"className":20856},[248],[20858,20863],{"type":150,"tag":169,"props":20859,"children":20861},{"className":20860},[248,489],[20862],{"type":155,"value":759},{"type":150,"tag":169,"props":20864,"children":20866},{"className":20865},[495],[20867],{"type":150,"tag":169,"props":20868,"children":20870},{"className":20869},[253,254],[20871,20900],{"type":150,"tag":169,"props":20872,"children":20874},{"className":20873},[259],[20875,20895],{"type":150,"tag":169,"props":20876,"children":20878},{"className":20877,"style":674},[264],[20879],{"type":150,"tag":169,"props":20880,"children":20881},{"style":512},[20882,20886],{"type":150,"tag":169,"props":20883,"children":20885},{"className":20884,"style":517},[274],[],{"type":150,"tag":169,"props":20887,"children":20889},{"className":20888},[522,523,524,525],[20890],{"type":150,"tag":169,"props":20891,"children":20893},{"className":20892,"style":2427},[248,489,525],[20894],{"type":155,"value":1876},{"type":150,"tag":169,"props":20896,"children":20898},{"className":20897},[355],[20899],{"type":155,"value":358},{"type":150,"tag":169,"props":20901,"children":20903},{"className":20902},[259],[20904],{"type":150,"tag":169,"props":20905,"children":20907},{"className":20906,"style":544},[264],[20908],{"type":150,"tag":169,"props":20909,"children":20910},{},[],{"type":150,"tag":169,"props":20912,"children":20914},{"className":20913},[248],[20915],{"type":155,"value":20916},"\u002F2",{"type":150,"tag":169,"props":20918,"children":20920},{"className":20919,"style":839},[558],[],{"type":150,"tag":169,"props":20922,"children":20924},{"className":20923},[844],[20925],{"type":155,"value":1045},{"type":150,"tag":169,"props":20927,"children":20929},{"className":20928,"style":839},[558],[],{"type":150,"tag":169,"props":20931,"children":20933},{"className":20932},[237],[20934,20938],{"type":150,"tag":169,"props":20935,"children":20937},{"className":20936,"style":5186},[242],[],{"type":150,"tag":169,"props":20939,"children":20941},{"className":20940},[248],[20942],{"type":155,"value":20836},{"type":155,"value":20944},"，相比 ",{"type":150,"tag":169,"props":20946,"children":20948},{"className":20947},[177],[20949,20987],{"type":150,"tag":169,"props":20950,"children":20952},{"className":20951},[182],[20953],{"type":150,"tag":185,"props":20954,"children":20955},{"xmlns":187},[20956],{"type":150,"tag":191,"props":20957,"children":20958},{},[20959,20982],{"type":150,"tag":195,"props":20960,"children":20961},{},[20962,20973,20977],{"type":150,"tag":401,"props":20963,"children":20964},{},[20965,20969],{"type":150,"tag":405,"props":20966,"children":20967},{},[20968],{"type":155,"value":759},{"type":150,"tag":405,"props":20970,"children":20971},{},[20972],{"type":155,"value":10046},{"type":150,"tag":417,"props":20974,"children":20975},{},[20976],{"type":155,"value":1045},{"type":150,"tag":411,"props":20978,"children":20979},{},[20980],{"type":155,"value":20981},"512",{"type":150,"tag":223,"props":20983,"children":20984},{"encoding":225},[20985],{"type":155,"value":20986},"d_c = 512",{"type":150,"tag":169,"props":20988,"children":20990},{"className":20989,"ariaHidden":214},[232],[20991,21069],{"type":150,"tag":169,"props":20992,"children":20994},{"className":20993},[237],[20995,20999,21056,21060,21065],{"type":150,"tag":169,"props":20996,"children":20998},{"className":20997,"style":4935},[242],[],{"type":150,"tag":169,"props":21000,"children":21002},{"className":21001},[248],[21003,21008],{"type":150,"tag":169,"props":21004,"children":21006},{"className":21005},[248,489],[21007],{"type":155,"value":759},{"type":150,"tag":169,"props":21009,"children":21011},{"className":21010},[495],[21012],{"type":150,"tag":169,"props":21013,"children":21015},{"className":21014},[253,254],[21016,21045],{"type":150,"tag":169,"props":21017,"children":21019},{"className":21018},[259],[21020,21040],{"type":150,"tag":169,"props":21021,"children":21023},{"className":21022,"style":5039},[264],[21024],{"type":150,"tag":169,"props":21025,"children":21026},{"style":512},[21027,21031],{"type":150,"tag":169,"props":21028,"children":21030},{"className":21029,"style":517},[274],[],{"type":150,"tag":169,"props":21032,"children":21034},{"className":21033},[522,523,524,525],[21035],{"type":150,"tag":169,"props":21036,"children":21038},{"className":21037},[248,489,525],[21039],{"type":155,"value":10046},{"type":150,"tag":169,"props":21041,"children":21043},{"className":21042},[355],[21044],{"type":155,"value":358},{"type":150,"tag":169,"props":21046,"children":21048},{"className":21047},[259],[21049],{"type":150,"tag":169,"props":21050,"children":21052},{"className":21051,"style":544},[264],[21053],{"type":150,"tag":169,"props":21054,"children":21055},{},[],{"type":150,"tag":169,"props":21057,"children":21059},{"className":21058,"style":839},[558],[],{"type":150,"tag":169,"props":21061,"children":21063},{"className":21062},[844],[21064],{"type":155,"value":1045},{"type":150,"tag":169,"props":21066,"children":21068},{"className":21067,"style":839},[558],[],{"type":150,"tag":169,"props":21070,"children":21072},{"className":21071},[237],[21073,21077],{"type":150,"tag":169,"props":21074,"children":21076},{"className":21075,"style":5186},[242],[],{"type":150,"tag":169,"props":21078,"children":21080},{"className":21079},[248],[21081],{"type":155,"value":20981},{"type":155,"value":21083},"，增加幅度不大。",{"type":150,"tag":151,"props":21085,"children":21086},{},[21087,21089,21219,21221,21356,21358,21553,21555,21657],{"type":155,"value":21088},"还有一个细节：MLA 最终版把 Q 的输入也改成了低秩投影。这跟减少 KV Cache 无关，主要是为了减少训练期间的参数量和梯度所占显存。",{"type":150,"tag":169,"props":21090,"children":21092},{"className":21091},[177],[21093,21128],{"type":150,"tag":169,"props":21094,"children":21096},{"className":21095},[182],[21097],{"type":150,"tag":185,"props":21098,"children":21099},{"xmlns":187},[21100],{"type":150,"tag":191,"props":21101,"children":21102},{},[21103,21123],{"type":150,"tag":195,"props":21104,"children":21105},{},[21106],{"type":150,"tag":1224,"props":21107,"children":21108},{},[21109,21113,21117],{"type":150,"tag":405,"props":21110,"children":21111},{},[21112],{"type":155,"value":759},{"type":150,"tag":405,"props":21114,"children":21115},{},[21116],{"type":155,"value":10046},{"type":150,"tag":417,"props":21118,"children":21120},{"mathvariant":1796,"lspace":21119,"rspace":21119},"0em",[21121],{"type":155,"value":21122},"′",{"type":150,"tag":223,"props":21124,"children":21125},{"encoding":225},[21126],{"type":155,"value":21127},"d_c'",{"type":150,"tag":169,"props":21129,"children":21131},{"className":21130,"ariaHidden":214},[232],[21132],{"type":150,"tag":169,"props":21133,"children":21135},{"className":21134},[237],[21136,21141],{"type":150,"tag":169,"props":21137,"children":21140},{"className":21138,"style":21139},[242],"height:0.9989em;vertical-align:-0.247em;",[],{"type":150,"tag":169,"props":21142,"children":21144},{"className":21143},[248],[21145,21150],{"type":150,"tag":169,"props":21146,"children":21148},{"className":21147},[248,489],[21149],{"type":155,"value":759},{"type":150,"tag":169,"props":21151,"children":21153},{"className":21152},[495],[21154],{"type":150,"tag":169,"props":21155,"children":21157},{"className":21156},[253,254],[21158,21208],{"type":150,"tag":169,"props":21159,"children":21161},{"className":21160},[259],[21162,21203],{"type":150,"tag":169,"props":21163,"children":21166},{"className":21164,"style":21165},[264],"height:0.7519em;",[21167,21183],{"type":150,"tag":169,"props":21168,"children":21169},{"style":15578},[21170,21174],{"type":150,"tag":169,"props":21171,"children":21173},{"className":21172,"style":517},[274],[],{"type":150,"tag":169,"props":21175,"children":21177},{"className":21176},[522,523,524,525],[21178],{"type":150,"tag":169,"props":21179,"children":21181},{"className":21180},[248,489,525],[21182],{"type":155,"value":10046},{"type":150,"tag":169,"props":21184,"children":21185},{"style":888},[21186,21190],{"type":150,"tag":169,"props":21187,"children":21189},{"className":21188,"style":517},[274],[],{"type":150,"tag":169,"props":21191,"children":21193},{"className":21192},[522,523,524,525],[21194],{"type":150,"tag":169,"props":21195,"children":21197},{"className":21196},[248,525],[21198],{"type":150,"tag":169,"props":21199,"children":21201},{"className":21200},[248,525],[21202],{"type":155,"value":21122},{"type":150,"tag":169,"props":21204,"children":21206},{"className":21205},[355],[21207],{"type":155,"value":358},{"type":150,"tag":169,"props":21209,"children":21211},{"className":21210},[259],[21212],{"type":150,"tag":169,"props":21213,"children":21215},{"className":21214,"style":4728},[264],[21216],{"type":150,"tag":169,"props":21217,"children":21218},{},[],{"type":155,"value":21220}," 取 1536，和 ",{"type":150,"tag":169,"props":21222,"children":21224},{"className":21223},[177],[21225,21261],{"type":150,"tag":169,"props":21226,"children":21228},{"className":21227},[182],[21229],{"type":150,"tag":185,"props":21230,"children":21231},{"xmlns":187},[21232],{"type":150,"tag":191,"props":21233,"children":21234},{},[21235,21257],{"type":150,"tag":195,"props":21236,"children":21237},{},[21238,21249,21253],{"type":150,"tag":401,"props":21239,"children":21240},{},[21241,21245],{"type":150,"tag":405,"props":21242,"children":21243},{},[21244],{"type":155,"value":759},{"type":150,"tag":405,"props":21246,"children":21247},{},[21248],{"type":155,"value":10046},{"type":150,"tag":417,"props":21250,"children":21251},{},[21252],{"type":155,"value":1045},{"type":150,"tag":411,"props":21254,"children":21255},{},[21256],{"type":155,"value":20981},{"type":150,"tag":223,"props":21258,"children":21259},{"encoding":225},[21260],{"type":155,"value":20986},{"type":150,"tag":169,"props":21262,"children":21264},{"className":21263,"ariaHidden":214},[232],[21265,21343],{"type":150,"tag":169,"props":21266,"children":21268},{"className":21267},[237],[21269,21273,21330,21334,21339],{"type":150,"tag":169,"props":21270,"children":21272},{"className":21271,"style":4935},[242],[],{"type":150,"tag":169,"props":21274,"children":21276},{"className":21275},[248],[21277,21282],{"type":150,"tag":169,"props":21278,"children":21280},{"className":21279},[248,489],[21281],{"type":155,"value":759},{"type":150,"tag":169,"props":21283,"children":21285},{"className":21284},[495],[21286],{"type":150,"tag":169,"props":21287,"children":21289},{"className":21288},[253,254],[21290,21319],{"type":150,"tag":169,"props":21291,"children":21293},{"className":21292},[259],[21294,21314],{"type":150,"tag":169,"props":21295,"children":21297},{"className":21296,"style":5039},[264],[21298],{"type":150,"tag":169,"props":21299,"children":21300},{"style":512},[21301,21305],{"type":150,"tag":169,"props":21302,"children":21304},{"className":21303,"style":517},[274],[],{"type":150,"tag":169,"props":21306,"children":21308},{"className":21307},[522,523,524,525],[21309],{"type":150,"tag":169,"props":21310,"children":21312},{"className":21311},[248,489,525],[21313],{"type":155,"value":10046},{"type":150,"tag":169,"props":21315,"children":21317},{"className":21316},[355],[21318],{"type":155,"value":358},{"type":150,"tag":169,"props":21320,"children":21322},{"className":21321},[259],[21323],{"type":150,"tag":169,"props":21324,"children":21326},{"className":21325,"style":544},[264],[21327],{"type":150,"tag":169,"props":21328,"children":21329},{},[],{"type":150,"tag":169,"props":21331,"children":21333},{"className":21332,"style":839},[558],[],{"type":150,"tag":169,"props":21335,"children":21337},{"className":21336},[844],[21338],{"type":155,"value":1045},{"type":150,"tag":169,"props":21340,"children":21342},{"className":21341,"style":839},[558],[],{"type":150,"tag":169,"props":21344,"children":21346},{"className":21345},[237],[21347,21351],{"type":150,"tag":169,"props":21348,"children":21350},{"className":21349,"style":5186},[242],[],{"type":150,"tag":169,"props":21352,"children":21354},{"className":21353},[248],[21355],{"type":155,"value":20981},{"type":155,"value":21357}," 不同。于是解码阶段的 MLA 是 MQA 形式，Q、K 的 head size 变成 ",{"type":150,"tag":169,"props":21359,"children":21361},{"className":21360},[177],[21362,21406],{"type":150,"tag":169,"props":21363,"children":21365},{"className":21364},[182],[21366],{"type":150,"tag":185,"props":21367,"children":21368},{"xmlns":187},[21369],{"type":150,"tag":191,"props":21370,"children":21371},{},[21372,21401],{"type":150,"tag":195,"props":21373,"children":21374},{},[21375,21386,21390],{"type":150,"tag":401,"props":21376,"children":21377},{},[21378,21382],{"type":150,"tag":405,"props":21379,"children":21380},{},[21381],{"type":155,"value":759},{"type":150,"tag":405,"props":21383,"children":21384},{},[21385],{"type":155,"value":10046},{"type":150,"tag":417,"props":21387,"children":21388},{},[21389],{"type":155,"value":5136},{"type":150,"tag":401,"props":21391,"children":21392},{},[21393,21397],{"type":150,"tag":405,"props":21394,"children":21395},{},[21396],{"type":155,"value":759},{"type":150,"tag":405,"props":21398,"children":21399},{},[21400],{"type":155,"value":20607},{"type":150,"tag":223,"props":21402,"children":21403},{"encoding":225},[21404],{"type":155,"value":21405},"d_c + d_r",{"type":150,"tag":169,"props":21407,"children":21409},{"className":21408,"ariaHidden":214},[232],[21410,21488],{"type":150,"tag":169,"props":21411,"children":21413},{"className":21412},[237],[21414,21418,21475,21479,21484],{"type":150,"tag":169,"props":21415,"children":21417},{"className":21416,"style":4935},[242],[],{"type":150,"tag":169,"props":21419,"children":21421},{"className":21420},[248],[21422,21427],{"type":150,"tag":169,"props":21423,"children":21425},{"className":21424},[248,489],[21426],{"type":155,"value":759},{"type":150,"tag":169,"props":21428,"children":21430},{"className":21429},[495],[21431],{"type":150,"tag":169,"props":21432,"children":21434},{"className":21433},[253,254],[21435,21464],{"type":150,"tag":169,"props":21436,"children":21438},{"className":21437},[259],[21439,21459],{"type":150,"tag":169,"props":21440,"children":21442},{"className":21441,"style":5039},[264],[21443],{"type":150,"tag":169,"props":21444,"children":21445},{"style":512},[21446,21450],{"type":150,"tag":169,"props":21447,"children":21449},{"className":21448,"style":517},[274],[],{"type":150,"tag":169,"props":21451,"children":21453},{"className":21452},[522,523,524,525],[21454],{"type":150,"tag":169,"props":21455,"children":21457},{"className":21456},[248,489,525],[21458],{"type":155,"value":10046},{"type":150,"tag":169,"props":21460,"children":21462},{"className":21461},[355],[21463],{"type":155,"value":358},{"type":150,"tag":169,"props":21465,"children":21467},{"className":21466},[259],[21468],{"type":150,"tag":169,"props":21469,"children":21471},{"className":21470,"style":544},[264],[21472],{"type":150,"tag":169,"props":21473,"children":21474},{},[],{"type":150,"tag":169,"props":21476,"children":21478},{"className":21477,"style":5167},[558],[],{"type":150,"tag":169,"props":21480,"children":21482},{"className":21481},[5172],[21483],{"type":155,"value":5136},{"type":150,"tag":169,"props":21485,"children":21487},{"className":21486,"style":5167},[558],[],{"type":150,"tag":169,"props":21489,"children":21491},{"className":21490},[237],[21492,21496],{"type":150,"tag":169,"props":21493,"children":21495},{"className":21494,"style":4935},[242],[],{"type":150,"tag":169,"props":21497,"children":21499},{"className":21498},[248],[21500,21505],{"type":150,"tag":169,"props":21501,"children":21503},{"className":21502},[248,489],[21504],{"type":155,"value":759},{"type":150,"tag":169,"props":21506,"children":21508},{"className":21507},[495],[21509],{"type":150,"tag":169,"props":21510,"children":21512},{"className":21511},[253,254],[21513,21542],{"type":150,"tag":169,"props":21514,"children":21516},{"className":21515},[259],[21517,21537],{"type":150,"tag":169,"props":21518,"children":21520},{"className":21519,"style":5039},[264],[21521],{"type":150,"tag":169,"props":21522,"children":21523},{"style":512},[21524,21528],{"type":150,"tag":169,"props":21525,"children":21527},{"className":21526,"style":517},[274],[],{"type":150,"tag":169,"props":21529,"children":21531},{"className":21530},[522,523,524,525],[21532],{"type":150,"tag":169,"props":21533,"children":21535},{"className":21534,"style":6968},[248,489,525],[21536],{"type":155,"value":20607},{"type":150,"tag":169,"props":21538,"children":21540},{"className":21539},[355],[21541],{"type":155,"value":358},{"type":150,"tag":169,"props":21543,"children":21545},{"className":21544},[259],[21546],{"type":150,"tag":169,"props":21547,"children":21549},{"className":21548,"style":544},[264],[21550],{"type":150,"tag":169,"props":21551,"children":21552},{},[],{"type":155,"value":21554},"，是 ",{"type":150,"tag":169,"props":21556,"children":21558},{"className":21557},[177],[21559,21588],{"type":150,"tag":169,"props":21560,"children":21562},{"className":21561},[182],[21563],{"type":150,"tag":185,"props":21564,"children":21565},{"xmlns":187},[21566],{"type":150,"tag":191,"props":21567,"children":21568},{},[21569,21583],{"type":150,"tag":195,"props":21570,"children":21571},{},[21572],{"type":150,"tag":401,"props":21573,"children":21574},{},[21575,21579],{"type":150,"tag":405,"props":21576,"children":21577},{},[21578],{"type":155,"value":759},{"type":150,"tag":405,"props":21580,"children":21581},{},[21582],{"type":155,"value":1876},{"type":150,"tag":223,"props":21584,"children":21585},{"encoding":225},[21586],{"type":155,"value":21587},"d_k",{"type":150,"tag":169,"props":21589,"children":21591},{"className":21590,"ariaHidden":214},[232],[21592],{"type":150,"tag":169,"props":21593,"children":21595},{"className":21594},[237],[21596,21600],{"type":150,"tag":169,"props":21597,"children":21599},{"className":21598,"style":4935},[242],[],{"type":150,"tag":169,"props":21601,"children":21603},{"className":21602},[248],[21604,21609],{"type":150,"tag":169,"props":21605,"children":21607},{"className":21606},[248,489],[21608],{"type":155,"value":759},{"type":150,"tag":169,"props":21610,"children":21612},{"className":21611},[495],[21613],{"type":150,"tag":169,"props":21614,"children":21616},{"className":21615},[253,254],[21617,21646],{"type":150,"tag":169,"props":21618,"children":21620},{"className":21619},[259],[21621,21641],{"type":150,"tag":169,"props":21622,"children":21624},{"className":21623,"style":674},[264],[21625],{"type":150,"tag":169,"props":21626,"children":21627},{"style":512},[21628,21632],{"type":150,"tag":169,"props":21629,"children":21631},{"className":21630,"style":517},[274],[],{"type":150,"tag":169,"props":21633,"children":21635},{"className":21634},[522,523,524,525],[21636],{"type":150,"tag":169,"props":21637,"children":21639},{"className":21638,"style":2427},[248,489,525],[21640],{"type":155,"value":1876},{"type":150,"tag":169,"props":21642,"children":21644},{"className":21643},[355],[21645],{"type":155,"value":358},{"type":150,"tag":169,"props":21647,"children":21649},{"className":21648},[259],[21650],{"type":150,"tag":169,"props":21651,"children":21653},{"className":21652,"style":544},[264],[21654],{"type":150,"tag":169,"props":21655,"children":21656},{},[],{"type":155,"value":21658}," 的 4 倍。",{"type":150,"tag":151,"props":21660,"children":21661},{},[21662],{"type":150,"tag":158,"props":21663,"children":21664},{},[21665],{"type":155,"value":21666},"解码的计算量其实是增加的。",{"type":150,"tag":151,"props":21668,"children":21669},{},[21670,21672,21750,21752,21829,21831,21967,21969,22104,22105,22241,22243,22321,22323,22365],{"type":155,"value":21671},"用 DeepSeek-V2 的配置把这笔账算一遍。",{"type":150,"tag":169,"props":21673,"children":21675},{"className":21674},[177],[21676,21707],{"type":150,"tag":169,"props":21677,"children":21679},{"className":21678},[182],[21680],{"type":150,"tag":185,"props":21681,"children":21682},{"xmlns":187},[21683],{"type":150,"tag":191,"props":21684,"children":21685},{},[21686,21702],{"type":150,"tag":195,"props":21687,"children":21688},{},[21689,21693,21697],{"type":150,"tag":405,"props":21690,"children":21691},{},[21692],{"type":155,"value":5559},{"type":150,"tag":417,"props":21694,"children":21695},{},[21696],{"type":155,"value":1045},{"type":150,"tag":411,"props":21698,"children":21699},{},[21700],{"type":155,"value":21701},"60",{"type":150,"tag":223,"props":21703,"children":21704},{"encoding":225},[21705],{"type":155,"value":21706},"L = 60",{"type":150,"tag":169,"props":21708,"children":21710},{"className":21709,"ariaHidden":214},[232],[21711,21737],{"type":150,"tag":169,"props":21712,"children":21714},{"className":21713},[237],[21715,21719,21724,21728,21733],{"type":150,"tag":169,"props":21716,"children":21718},{"className":21717,"style":5854},[242],[],{"type":150,"tag":169,"props":21720,"children":21722},{"className":21721},[248,489],[21723],{"type":155,"value":5559},{"type":150,"tag":169,"props":21725,"children":21727},{"className":21726,"style":839},[558],[],{"type":150,"tag":169,"props":21729,"children":21731},{"className":21730},[844],[21732],{"type":155,"value":1045},{"type":150,"tag":169,"props":21734,"children":21736},{"className":21735,"style":839},[558],[],{"type":150,"tag":169,"props":21738,"children":21740},{"className":21739},[237],[21741,21745],{"type":150,"tag":169,"props":21742,"children":21744},{"className":21743,"style":5186},[242],[],{"type":150,"tag":169,"props":21746,"children":21748},{"className":21747},[248],[21749],{"type":155,"value":21701},{"type":155,"value":21751}," 层，",{"type":150,"tag":169,"props":21753,"children":21755},{"className":21754},[177],[21756,21786],{"type":150,"tag":169,"props":21757,"children":21759},{"className":21758},[182],[21760],{"type":150,"tag":185,"props":21761,"children":21762},{"xmlns":187},[21763],{"type":150,"tag":191,"props":21764,"children":21765},{},[21766,21781],{"type":150,"tag":195,"props":21767,"children":21768},{},[21769,21773,21777],{"type":150,"tag":405,"props":21770,"children":21771},{},[21772],{"type":155,"value":926},{"type":150,"tag":417,"props":21774,"children":21775},{},[21776],{"type":155,"value":1045},{"type":150,"tag":411,"props":21778,"children":21779},{},[21780],{"type":155,"value":6045},{"type":150,"tag":223,"props":21782,"children":21783},{"encoding":225},[21784],{"type":155,"value":21785},"h = 128",{"type":150,"tag":169,"props":21787,"children":21789},{"className":21788,"ariaHidden":214},[232],[21790,21816],{"type":150,"tag":169,"props":21791,"children":21793},{"className":21792},[237],[21794,21798,21803,21807,21812],{"type":150,"tag":169,"props":21795,"children":21797},{"className":21796,"style":942},[242],[],{"type":150,"tag":169,"props":21799,"children":21801},{"className":21800},[248,489],[21802],{"type":155,"value":926},{"type":150,"tag":169,"props":21804,"children":21806},{"className":21805,"style":839},[558],[],{"type":150,"tag":169,"props":21808,"children":21810},{"className":21809},[844],[21811],{"type":155,"value":1045},{"type":150,"tag":169,"props":21813,"children":21815},{"className":21814,"style":839},[558],[],{"type":150,"tag":169,"props":21817,"children":21819},{"className":21818},[237],[21820,21824],{"type":150,"tag":169,"props":21821,"children":21823},{"className":21822,"style":5186},[242],[],{"type":150,"tag":169,"props":21825,"children":21827},{"className":21826},[248],[21828],{"type":155,"value":6045},{"type":155,"value":21830}," 个 head，",{"type":150,"tag":169,"props":21832,"children":21834},{"className":21833},[177],[21835,21872],{"type":150,"tag":169,"props":21836,"children":21838},{"className":21837},[182],[21839],{"type":150,"tag":185,"props":21840,"children":21841},{"xmlns":187},[21842],{"type":150,"tag":191,"props":21843,"children":21844},{},[21845,21867],{"type":150,"tag":195,"props":21846,"children":21847},{},[21848,21859,21863],{"type":150,"tag":401,"props":21849,"children":21850},{},[21851,21855],{"type":150,"tag":405,"props":21852,"children":21853},{},[21854],{"type":155,"value":759},{"type":150,"tag":405,"props":21856,"children":21857},{},[21858],{"type":155,"value":1876},{"type":150,"tag":417,"props":21860,"children":21861},{},[21862],{"type":155,"value":1045},{"type":150,"tag":411,"props":21864,"children":21865},{},[21866],{"type":155,"value":6045},{"type":150,"tag":223,"props":21868,"children":21869},{"encoding":225},[21870],{"type":155,"value":21871},"d_k = 128",{"type":150,"tag":169,"props":21873,"children":21875},{"className":21874,"ariaHidden":214},[232],[21876,21954],{"type":150,"tag":169,"props":21877,"children":21879},{"className":21878},[237],[21880,21884,21941,21945,21950],{"type":150,"tag":169,"props":21881,"children":21883},{"className":21882,"style":4935},[242],[],{"type":150,"tag":169,"props":21885,"children":21887},{"className":21886},[248],[21888,21893],{"type":150,"tag":169,"props":21889,"children":21891},{"className":21890},[248,489],[21892],{"type":155,"value":759},{"type":150,"tag":169,"props":21894,"children":21896},{"className":21895},[495],[21897],{"type":150,"tag":169,"props":21898,"children":21900},{"className":21899},[253,254],[21901,21930],{"type":150,"tag":169,"props":21902,"children":21904},{"className":21903},[259],[21905,21925],{"type":150,"tag":169,"props":21906,"children":21908},{"className":21907,"style":674},[264],[21909],{"type":150,"tag":169,"props":21910,"children":21911},{"style":512},[21912,21916],{"type":150,"tag":169,"props":21913,"children":21915},{"className":21914,"style":517},[274],[],{"type":150,"tag":169,"props":21917,"children":21919},{"className":21918},[522,523,524,525],[21920],{"type":150,"tag":169,"props":21921,"children":21923},{"className":21922,"style":2427},[248,489,525],[21924],{"type":155,"value":1876},{"type":150,"tag":169,"props":21926,"children":21928},{"className":21927},[355],[21929],{"type":155,"value":358},{"type":150,"tag":169,"props":21931,"children":21933},{"className":21932},[259],[21934],{"type":150,"tag":169,"props":21935,"children":21937},{"className":21936,"style":544},[264],[21938],{"type":150,"tag":169,"props":21939,"children":21940},{},[],{"type":150,"tag":169,"props":21942,"children":21944},{"className":21943,"style":839},[558],[],{"type":150,"tag":169,"props":21946,"children":21948},{"className":21947},[844],[21949],{"type":155,"value":1045},{"type":150,"tag":169,"props":21951,"children":21953},{"className":21952,"style":839},[558],[],{"type":150,"tag":169,"props":21955,"children":21957},{"className":21956},[237],[21958,21962],{"type":150,"tag":169,"props":21959,"children":21961},{"className":21960,"style":5186},[242],[],{"type":150,"tag":169,"props":21963,"children":21965},{"className":21964},[248],[21966],{"type":155,"value":6045},{"type":155,"value":21968},"，MLA 这边 ",{"type":150,"tag":169,"props":21970,"children":21972},{"className":21971},[177],[21973,22009],{"type":150,"tag":169,"props":21974,"children":21976},{"className":21975},[182],[21977],{"type":150,"tag":185,"props":21978,"children":21979},{"xmlns":187},[21980],{"type":150,"tag":191,"props":21981,"children":21982},{},[21983,22005],{"type":150,"tag":195,"props":21984,"children":21985},{},[21986,21997,22001],{"type":150,"tag":401,"props":21987,"children":21988},{},[21989,21993],{"type":150,"tag":405,"props":21990,"children":21991},{},[21992],{"type":155,"value":759},{"type":150,"tag":405,"props":21994,"children":21995},{},[21996],{"type":155,"value":10046},{"type":150,"tag":417,"props":21998,"children":21999},{},[22000],{"type":155,"value":1045},{"type":150,"tag":411,"props":22002,"children":22003},{},[22004],{"type":155,"value":20981},{"type":150,"tag":223,"props":22006,"children":22007},{"encoding":225},[22008],{"type":155,"value":20986},{"type":150,"tag":169,"props":22010,"children":22012},{"className":22011,"ariaHidden":214},[232],[22013,22091],{"type":150,"tag":169,"props":22014,"children":22016},{"className":22015},[237],[22017,22021,22078,22082,22087],{"type":150,"tag":169,"props":22018,"children":22020},{"className":22019,"style":4935},[242],[],{"type":150,"tag":169,"props":22022,"children":22024},{"className":22023},[248],[22025,22030],{"type":150,"tag":169,"props":22026,"children":22028},{"className":22027},[248,489],[22029],{"type":155,"value":759},{"type":150,"tag":169,"props":22031,"children":22033},{"className":22032},[495],[22034],{"type":150,"tag":169,"props":22035,"children":22037},{"className":22036},[253,254],[22038,22067],{"type":150,"tag":169,"props":22039,"children":22041},{"className":22040},[259],[22042,22062],{"type":150,"tag":169,"props":22043,"children":22045},{"className":22044,"style":5039},[264],[22046],{"type":150,"tag":169,"props":22047,"children":22048},{"style":512},[22049,22053],{"type":150,"tag":169,"props":22050,"children":22052},{"className":22051,"style":517},[274],[],{"type":150,"tag":169,"props":22054,"children":22056},{"className":22055},[522,523,524,525],[22057],{"type":150,"tag":169,"props":22058,"children":22060},{"className":22059},[248,489,525],[22061],{"type":155,"value":10046},{"type":150,"tag":169,"props":22063,"children":22065},{"className":22064},[355],[22066],{"type":155,"value":358},{"type":150,"tag":169,"props":22068,"children":22070},{"className":22069},[259],[22071],{"type":150,"tag":169,"props":22072,"children":22074},{"className":22073,"style":544},[264],[22075],{"type":150,"tag":169,"props":22076,"children":22077},{},[],{"type":150,"tag":169,"props":22079,"children":22081},{"className":22080,"style":839},[558],[],{"type":150,"tag":169,"props":22083,"children":22085},{"className":22084},[844],[22086],{"type":155,"value":1045},{"type":150,"tag":169,"props":22088,"children":22090},{"className":22089,"style":839},[558],[],{"type":150,"tag":169,"props":22092,"children":22094},{"className":22093},[237],[22095,22099],{"type":150,"tag":169,"props":22096,"children":22098},{"className":22097,"style":5186},[242],[],{"type":150,"tag":169,"props":22100,"children":22102},{"className":22101},[248],[22103],{"type":155,"value":20981},{"type":155,"value":9439},{"type":150,"tag":169,"props":22106,"children":22108},{"className":22107},[177],[22109,22146],{"type":150,"tag":169,"props":22110,"children":22112},{"className":22111},[182],[22113],{"type":150,"tag":185,"props":22114,"children":22115},{"xmlns":187},[22116],{"type":150,"tag":191,"props":22117,"children":22118},{},[22119,22141],{"type":150,"tag":195,"props":22120,"children":22121},{},[22122,22133,22137],{"type":150,"tag":401,"props":22123,"children":22124},{},[22125,22129],{"type":150,"tag":405,"props":22126,"children":22127},{},[22128],{"type":155,"value":759},{"type":150,"tag":405,"props":22130,"children":22131},{},[22132],{"type":155,"value":20607},{"type":150,"tag":417,"props":22134,"children":22135},{},[22136],{"type":155,"value":1045},{"type":150,"tag":411,"props":22138,"children":22139},{},[22140],{"type":155,"value":20836},{"type":150,"tag":223,"props":22142,"children":22143},{"encoding":225},[22144],{"type":155,"value":22145},"d_r = 64",{"type":150,"tag":169,"props":22147,"children":22149},{"className":22148,"ariaHidden":214},[232],[22150,22228],{"type":150,"tag":169,"props":22151,"children":22153},{"className":22152},[237],[22154,22158,22215,22219,22224],{"type":150,"tag":169,"props":22155,"children":22157},{"className":22156,"style":4935},[242],[],{"type":150,"tag":169,"props":22159,"children":22161},{"className":22160},[248],[22162,22167],{"type":150,"tag":169,"props":22163,"children":22165},{"className":22164},[248,489],[22166],{"type":155,"value":759},{"type":150,"tag":169,"props":22168,"children":22170},{"className":22169},[495],[22171],{"type":150,"tag":169,"props":22172,"children":22174},{"className":22173},[253,254],[22175,22204],{"type":150,"tag":169,"props":22176,"children":22178},{"className":22177},[259],[22179,22199],{"type":150,"tag":169,"props":22180,"children":22182},{"className":22181,"style":5039},[264],[22183],{"type":150,"tag":169,"props":22184,"children":22185},{"style":512},[22186,22190],{"type":150,"tag":169,"props":22187,"children":22189},{"className":22188,"style":517},[274],[],{"type":150,"tag":169,"props":22191,"children":22193},{"className":22192},[522,523,524,525],[22194],{"type":150,"tag":169,"props":22195,"children":22197},{"className":22196,"style":6968},[248,489,525],[22198],{"type":155,"value":20607},{"type":150,"tag":169,"props":22200,"children":22202},{"className":22201},[355],[22203],{"type":155,"value":358},{"type":150,"tag":169,"props":22205,"children":22207},{"className":22206},[259],[22208],{"type":150,"tag":169,"props":22209,"children":22211},{"className":22210,"style":544},[264],[22212],{"type":150,"tag":169,"props":22213,"children":22214},{},[],{"type":150,"tag":169,"props":22216,"children":22218},{"className":22217,"style":839},[558],[],{"type":150,"tag":169,"props":22220,"children":22222},{"className":22221},[844],[22223],{"type":155,"value":1045},{"type":150,"tag":169,"props":22225,"children":22227},{"className":22226,"style":839},[558],[],{"type":150,"tag":169,"props":22229,"children":22231},{"className":22230},[237],[22232,22236],{"type":150,"tag":169,"props":22233,"children":22235},{"className":22234,"style":5186},[242],[],{"type":150,"tag":169,"props":22237,"children":22239},{"className":22238},[248],[22240],{"type":155,"value":20836},{"type":155,"value":22242},"，BF16 存储，所以每参数 ",{"type":150,"tag":169,"props":22244,"children":22246},{"className":22245},[177],[22247,22278],{"type":150,"tag":169,"props":22248,"children":22250},{"className":22249},[182],[22251],{"type":150,"tag":185,"props":22252,"children":22253},{"xmlns":187},[22254],{"type":150,"tag":191,"props":22255,"children":22256},{},[22257,22273],{"type":150,"tag":195,"props":22258,"children":22259},{},[22260,22265,22269],{"type":150,"tag":405,"props":22261,"children":22262},{},[22263],{"type":155,"value":22264},"b",{"type":150,"tag":417,"props":22266,"children":22267},{},[22268],{"type":155,"value":1045},{"type":150,"tag":411,"props":22270,"children":22271},{},[22272],{"type":155,"value":433},{"type":150,"tag":223,"props":22274,"children":22275},{"encoding":225},[22276],{"type":155,"value":22277},"b = 2",{"type":150,"tag":169,"props":22279,"children":22281},{"className":22280,"ariaHidden":214},[232],[22282,22308],{"type":150,"tag":169,"props":22283,"children":22285},{"className":22284},[237],[22286,22290,22295,22299,22304],{"type":150,"tag":169,"props":22287,"children":22289},{"className":22288,"style":942},[242],[],{"type":150,"tag":169,"props":22291,"children":22293},{"className":22292},[248,489],[22294],{"type":155,"value":22264},{"type":150,"tag":169,"props":22296,"children":22298},{"className":22297,"style":839},[558],[],{"type":150,"tag":169,"props":22300,"children":22302},{"className":22301},[844],[22303],{"type":155,"value":1045},{"type":150,"tag":169,"props":22305,"children":22307},{"className":22306,"style":839},[558],[],{"type":150,"tag":169,"props":22309,"children":22311},{"className":22310},[237],[22312,22316],{"type":150,"tag":169,"props":22313,"children":22315},{"className":22314,"style":5186},[242],[],{"type":150,"tag":169,"props":22317,"children":22319},{"className":22318},[248],[22320],{"type":155,"value":433},{"type":155,"value":22322}," 字节。decode 到第 ",{"type":150,"tag":169,"props":22324,"children":22326},{"className":22325},[177],[22327,22348],{"type":150,"tag":169,"props":22328,"children":22330},{"className":22329},[182],[22331],{"type":150,"tag":185,"props":22332,"children":22333},{"xmlns":187},[22334],{"type":150,"tag":191,"props":22335,"children":22336},{},[22337,22344],{"type":150,"tag":195,"props":22338,"children":22339},{},[22340],{"type":150,"tag":405,"props":22341,"children":22342},{},[22343],{"type":155,"value":5591},{"type":150,"tag":223,"props":22345,"children":22346},{"encoding":225},[22347],{"type":155,"value":5591},{"type":150,"tag":169,"props":22349,"children":22351},{"className":22350,"ariaHidden":214},[232],[22352],{"type":150,"tag":169,"props":22353,"children":22355},{"className":22354},[237],[22356,22360],{"type":150,"tag":169,"props":22357,"children":22359},{"className":22358,"style":5854},[242],[],{"type":150,"tag":169,"props":22361,"children":22363},{"className":22362,"style":5780},[248,489],[22364],{"type":155,"value":5591},{"type":155,"value":22366}," 个位置时，每 token 每层：",{"type":150,"tag":22368,"props":22369,"children":22370},"table",{},[22371,22393],{"type":150,"tag":22372,"props":22373,"children":22374},"thead",{},[22375],{"type":150,"tag":22376,"props":22377,"children":22378},"tr",{},[22379,22383,22388],{"type":150,"tag":22380,"props":22381,"children":22382},"th",{},[],{"type":150,"tag":22380,"props":22384,"children":22385},{},[22386],{"type":155,"value":22387},"KV 读取",{"type":150,"tag":22380,"props":22389,"children":22390},{},[22391],{"type":155,"value":22392},"attention FLOPs",{"type":150,"tag":22394,"props":22395,"children":22396},"tbody",{},[22397,22680],{"type":150,"tag":22376,"props":22398,"children":22399},{},[22400,22406,22547],{"type":150,"tag":22401,"props":22402,"children":22403},"td",{},[22404],{"type":155,"value":22405},"MHA",{"type":150,"tag":22401,"props":22407,"children":22408},{},[22409],{"type":150,"tag":169,"props":22410,"children":22412},{"className":22411},[177],[22413,22458],{"type":150,"tag":169,"props":22414,"children":22416},{"className":22415},[182],[22417],{"type":150,"tag":185,"props":22418,"children":22419},{"xmlns":187},[22420],{"type":150,"tag":191,"props":22421,"children":22422},{},[22423,22453],{"type":150,"tag":195,"props":22424,"children":22425},{},[22426,22430,22434,22445,22449],{"type":150,"tag":411,"props":22427,"children":22428},{},[22429],{"type":155,"value":433},{"type":150,"tag":405,"props":22431,"children":22432},{},[22433],{"type":155,"value":926},{"type":150,"tag":401,"props":22435,"children":22436},{},[22437,22441],{"type":150,"tag":405,"props":22438,"children":22439},{},[22440],{"type":155,"value":759},{"type":150,"tag":405,"props":22442,"children":22443},{},[22444],{"type":155,"value":1876},{"type":150,"tag":405,"props":22446,"children":22447},{},[22448],{"type":155,"value":5591},{"type":150,"tag":405,"props":22450,"children":22451},{},[22452],{"type":155,"value":22264},{"type":150,"tag":223,"props":22454,"children":22455},{"encoding":225},[22456],{"type":155,"value":22457},"2 h d_k N b",{"type":150,"tag":169,"props":22459,"children":22461},{"className":22460,"ariaHidden":214},[232],[22462],{"type":150,"tag":169,"props":22463,"children":22465},{"className":22464},[237],[22466,22470,22475,22480,22537,22542],{"type":150,"tag":169,"props":22467,"children":22469},{"className":22468,"style":4935},[242],[],{"type":150,"tag":169,"props":22471,"children":22473},{"className":22472},[248],[22474],{"type":155,"value":433},{"type":150,"tag":169,"props":22476,"children":22478},{"className":22477},[248,489],[22479],{"type":155,"value":926},{"type":150,"tag":169,"props":22481,"children":22483},{"className":22482},[248],[22484,22489],{"type":150,"tag":169,"props":22485,"children":22487},{"className":22486},[248,489],[22488],{"type":155,"value":759},{"type":150,"tag":169,"props":22490,"children":22492},{"className":22491},[495],[22493],{"type":150,"tag":169,"props":22494,"children":22496},{"className":22495},[253,254],[22497,22526],{"type":150,"tag":169,"props":22498,"children":22500},{"className":22499},[259],[22501,22521],{"type":150,"tag":169,"props":22502,"children":22504},{"className":22503,"style":674},[264],[22505],{"type":150,"tag":169,"props":22506,"children":22507},{"style":512},[22508,22512],{"type":150,"tag":169,"props":22509,"children":22511},{"className":22510,"style":517},[274],[],{"type":150,"tag":169,"props":22513,"children":22515},{"className":22514},[522,523,524,525],[22516],{"type":150,"tag":169,"props":22517,"children":22519},{"className":22518,"style":2427},[248,489,525],[22520],{"type":155,"value":1876},{"type":150,"tag":169,"props":22522,"children":22524},{"className":22523},[355],[22525],{"type":155,"value":358},{"type":150,"tag":169,"props":22527,"children":22529},{"className":22528},[259],[22530],{"type":150,"tag":169,"props":22531,"children":22533},{"className":22532,"style":544},[264],[22534],{"type":150,"tag":169,"props":22535,"children":22536},{},[],{"type":150,"tag":169,"props":22538,"children":22540},{"className":22539,"style":5780},[248,489],[22541],{"type":155,"value":5591},{"type":150,"tag":169,"props":22543,"children":22545},{"className":22544},[248,489],[22546],{"type":155,"value":22264},{"type":150,"tag":22401,"props":22548,"children":22549},{},[22550],{"type":150,"tag":169,"props":22551,"children":22553},{"className":22552},[177],[22554,22596],{"type":150,"tag":169,"props":22555,"children":22557},{"className":22556},[182],[22558],{"type":150,"tag":185,"props":22559,"children":22560},{"xmlns":187},[22561],{"type":150,"tag":191,"props":22562,"children":22563},{},[22564,22591],{"type":150,"tag":195,"props":22565,"children":22566},{},[22567,22572,22576,22587],{"type":150,"tag":411,"props":22568,"children":22569},{},[22570],{"type":155,"value":22571},"4",{"type":150,"tag":405,"props":22573,"children":22574},{},[22575],{"type":155,"value":926},{"type":150,"tag":401,"props":22577,"children":22578},{},[22579,22583],{"type":150,"tag":405,"props":22580,"children":22581},{},[22582],{"type":155,"value":759},{"type":150,"tag":405,"props":22584,"children":22585},{},[22586],{"type":155,"value":1876},{"type":150,"tag":405,"props":22588,"children":22589},{},[22590],{"type":155,"value":5591},{"type":150,"tag":223,"props":22592,"children":22593},{"encoding":225},[22594],{"type":155,"value":22595},"4 h d_k N",{"type":150,"tag":169,"props":22597,"children":22599},{"className":22598,"ariaHidden":214},[232],[22600],{"type":150,"tag":169,"props":22601,"children":22603},{"className":22602},[237],[22604,22608,22613,22618,22675],{"type":150,"tag":169,"props":22605,"children":22607},{"className":22606,"style":4935},[242],[],{"type":150,"tag":169,"props":22609,"children":22611},{"className":22610},[248],[22612],{"type":155,"value":22571},{"type":150,"tag":169,"props":22614,"children":22616},{"className":22615},[248,489],[22617],{"type":155,"value":926},{"type":150,"tag":169,"props":22619,"children":22621},{"className":22620},[248],[22622,22627],{"type":150,"tag":169,"props":22623,"children":22625},{"className":22624},[248,489],[22626],{"type":155,"value":759},{"type":150,"tag":169,"props":22628,"children":22630},{"className":22629},[495],[22631],{"type":150,"tag":169,"props":22632,"children":22634},{"className":22633},[253,254],[22635,22664],{"type":150,"tag":169,"props":22636,"children":22638},{"className":22637},[259],[22639,22659],{"type":150,"tag":169,"props":22640,"children":22642},{"className":22641,"style":674},[264],[22643],{"type":150,"tag":169,"props":22644,"children":22645},{"style":512},[22646,22650],{"type":150,"tag":169,"props":22647,"children":22649},{"className":22648,"style":517},[274],[],{"type":150,"tag":169,"props":22651,"children":22653},{"className":22652},[522,523,524,525],[22654],{"type":150,"tag":169,"props":22655,"children":22657},{"className":22656,"style":2427},[248,489,525],[22658],{"type":155,"value":1876},{"type":150,"tag":169,"props":22660,"children":22662},{"className":22661},[355],[22663],{"type":155,"value":358},{"type":150,"tag":169,"props":22665,"children":22667},{"className":22666},[259],[22668],{"type":150,"tag":169,"props":22669,"children":22671},{"className":22670,"style":544},[264],[22672],{"type":150,"tag":169,"props":22673,"children":22674},{},[],{"type":150,"tag":169,"props":22676,"children":22678},{"className":22677,"style":5780},[248,489],[22679],{"type":155,"value":5591},{"type":150,"tag":22376,"props":22681,"children":22682},{},[22683,22688,22922],{"type":150,"tag":22401,"props":22684,"children":22685},{},[22686],{"type":155,"value":22687},"MLA decode",{"type":150,"tag":22401,"props":22689,"children":22690},{},[22691],{"type":150,"tag":169,"props":22692,"children":22694},{"className":22693},[177],[22695,22755],{"type":150,"tag":169,"props":22696,"children":22698},{"className":22697},[182],[22699],{"type":150,"tag":185,"props":22700,"children":22701},{"xmlns":187},[22702],{"type":150,"tag":191,"props":22703,"children":22704},{},[22705,22750],{"type":150,"tag":195,"props":22706,"children":22707},{},[22708,22712,22723,22727,22738,22742,22746],{"type":150,"tag":417,"props":22709,"children":22710},{"stretchy":207},[22711],{"type":155,"value":972},{"type":150,"tag":401,"props":22713,"children":22714},{},[22715,22719],{"type":150,"tag":405,"props":22716,"children":22717},{},[22718],{"type":155,"value":759},{"type":150,"tag":405,"props":22720,"children":22721},{},[22722],{"type":155,"value":10046},{"type":150,"tag":417,"props":22724,"children":22725},{},[22726],{"type":155,"value":5136},{"type":150,"tag":401,"props":22728,"children":22729},{},[22730,22734],{"type":150,"tag":405,"props":22731,"children":22732},{},[22733],{"type":155,"value":759},{"type":150,"tag":405,"props":22735,"children":22736},{},[22737],{"type":155,"value":20607},{"type":150,"tag":417,"props":22739,"children":22740},{"stretchy":207},[22741],{"type":155,"value":982},{"type":150,"tag":405,"props":22743,"children":22744},{},[22745],{"type":155,"value":5591},{"type":150,"tag":405,"props":22747,"children":22748},{},[22749],{"type":155,"value":22264},{"type":150,"tag":223,"props":22751,"children":22752},{"encoding":225},[22753],{"type":155,"value":22754},"(d_c + d_r) N b",{"type":150,"tag":169,"props":22756,"children":22758},{"className":22757,"ariaHidden":214},[232],[22759,22842],{"type":150,"tag":169,"props":22760,"children":22762},{"className":22761},[237],[22763,22767,22772,22829,22833,22838],{"type":150,"tag":169,"props":22764,"children":22766},{"className":22765,"style":999},[242],[],{"type":150,"tag":169,"props":22768,"children":22770},{"className":22769},[1004],[22771],{"type":155,"value":972},{"type":150,"tag":169,"props":22773,"children":22775},{"className":22774},[248],[22776,22781],{"type":150,"tag":169,"props":22777,"children":22779},{"className":22778},[248,489],[22780],{"type":155,"value":759},{"type":150,"tag":169,"props":22782,"children":22784},{"className":22783},[495],[22785],{"type":150,"tag":169,"props":22786,"children":22788},{"className":22787},[253,254],[22789,22818],{"type":150,"tag":169,"props":22790,"children":22792},{"className":22791},[259],[22793,22813],{"type":150,"tag":169,"props":22794,"children":22796},{"className":22795,"style":5039},[264],[22797],{"type":150,"tag":169,"props":22798,"children":22799},{"style":512},[22800,22804],{"type":150,"tag":169,"props":22801,"children":22803},{"className":22802,"style":517},[274],[],{"type":150,"tag":169,"props":22805,"children":22807},{"className":22806},[522,523,524,525],[22808],{"type":150,"tag":169,"props":22809,"children":22811},{"className":22810},[248,489,525],[22812],{"type":155,"value":10046},{"type":150,"tag":169,"props":22814,"children":22816},{"className":22815},[355],[22817],{"type":155,"value":358},{"type":150,"tag":169,"props":22819,"children":22821},{"className":22820},[259],[22822],{"type":150,"tag":169,"props":22823,"children":22825},{"className":22824,"style":544},[264],[22826],{"type":150,"tag":169,"props":22827,"children":22828},{},[],{"type":150,"tag":169,"props":22830,"children":22832},{"className":22831,"style":5167},[558],[],{"type":150,"tag":169,"props":22834,"children":22836},{"className":22835},[5172],[22837],{"type":155,"value":5136},{"type":150,"tag":169,"props":22839,"children":22841},{"className":22840,"style":5167},[558],[],{"type":150,"tag":169,"props":22843,"children":22845},{"className":22844},[237],[22846,22850,22907,22912,22917],{"type":150,"tag":169,"props":22847,"children":22849},{"className":22848,"style":999},[242],[],{"type":150,"tag":169,"props":22851,"children":22853},{"className":22852},[248],[22854,22859],{"type":150,"tag":169,"props":22855,"children":22857},{"className":22856},[248,489],[22858],{"type":155,"value":759},{"type":150,"tag":169,"props":22860,"children":22862},{"className":22861},[495],[22863],{"type":150,"tag":169,"props":22864,"children":22866},{"className":22865},[253,254],[22867,22896],{"type":150,"tag":169,"props":22868,"children":22870},{"className":22869},[259],[22871,22891],{"type":150,"tag":169,"props":22872,"children":22874},{"className":22873,"style":5039},[264],[22875],{"type":150,"tag":169,"props":22876,"children":22877},{"style":512},[22878,22882],{"type":150,"tag":169,"props":22879,"children":22881},{"className":22880,"style":517},[274],[],{"type":150,"tag":169,"props":22883,"children":22885},{"className":22884},[522,523,524,525],[22886],{"type":150,"tag":169,"props":22887,"children":22889},{"className":22888,"style":6968},[248,489,525],[22890],{"type":155,"value":20607},{"type":150,"tag":169,"props":22892,"children":22894},{"className":22893},[355],[22895],{"type":155,"value":358},{"type":150,"tag":169,"props":22897,"children":22899},{"className":22898},[259],[22900],{"type":150,"tag":169,"props":22901,"children":22903},{"className":22902,"style":544},[264],[22904],{"type":150,"tag":169,"props":22905,"children":22906},{},[],{"type":150,"tag":169,"props":22908,"children":22910},{"className":22909},[1015],[22911],{"type":155,"value":982},{"type":150,"tag":169,"props":22913,"children":22915},{"className":22914,"style":5780},[248,489],[22916],{"type":155,"value":5591},{"type":150,"tag":169,"props":22918,"children":22920},{"className":22919},[248,489],[22921],{"type":155,"value":22264},{"type":150,"tag":22401,"props":22923,"children":22924},{},[22925],{"type":150,"tag":169,"props":22926,"children":22928},{"className":22927},[177],[22929,23020],{"type":150,"tag":169,"props":22930,"children":22932},{"className":22931},[182],[22933],{"type":150,"tag":185,"props":22934,"children":22935},{"xmlns":187},[22936],{"type":150,"tag":191,"props":22937,"children":22938},{},[22939,23015],{"type":150,"tag":195,"props":22940,"children":22941},{},[22942,22946,22950,22954,22965,22969,22980,22984,22988,22992,22996,23000,23011],{"type":150,"tag":411,"props":22943,"children":22944},{},[22945],{"type":155,"value":433},{"type":150,"tag":405,"props":22947,"children":22948},{},[22949],{"type":155,"value":926},{"type":150,"tag":417,"props":22951,"children":22952},{"stretchy":207},[22953],{"type":155,"value":972},{"type":150,"tag":401,"props":22955,"children":22956},{},[22957,22961],{"type":150,"tag":405,"props":22958,"children":22959},{},[22960],{"type":155,"value":759},{"type":150,"tag":405,"props":22962,"children":22963},{},[22964],{"type":155,"value":10046},{"type":150,"tag":417,"props":22966,"children":22967},{},[22968],{"type":155,"value":5136},{"type":150,"tag":401,"props":22970,"children":22971},{},[22972,22976],{"type":150,"tag":405,"props":22973,"children":22974},{},[22975],{"type":155,"value":759},{"type":150,"tag":405,"props":22977,"children":22978},{},[22979],{"type":155,"value":20607},{"type":150,"tag":417,"props":22981,"children":22982},{"stretchy":207},[22983],{"type":155,"value":982},{"type":150,"tag":405,"props":22985,"children":22986},{},[22987],{"type":155,"value":5591},{"type":150,"tag":417,"props":22989,"children":22990},{},[22991],{"type":155,"value":5136},{"type":150,"tag":411,"props":22993,"children":22994},{},[22995],{"type":155,"value":433},{"type":150,"tag":405,"props":22997,"children":22998},{},[22999],{"type":155,"value":926},{"type":150,"tag":401,"props":23001,"children":23002},{},[23003,23007],{"type":150,"tag":405,"props":23004,"children":23005},{},[23006],{"type":155,"value":759},{"type":150,"tag":405,"props":23008,"children":23009},{},[23010],{"type":155,"value":10046},{"type":150,"tag":405,"props":23012,"children":23013},{},[23014],{"type":155,"value":5591},{"type":150,"tag":223,"props":23016,"children":23017},{"encoding":225},[23018],{"type":155,"value":23019},"2h(d_c+d_r)N + 2hd_cN",{"type":150,"tag":169,"props":23021,"children":23023},{"className":23022,"ariaHidden":214},[232],[23024,23117,23205],{"type":150,"tag":169,"props":23025,"children":23027},{"className":23026},[237],[23028,23032,23037,23042,23047,23104,23108,23113],{"type":150,"tag":169,"props":23029,"children":23031},{"className":23030,"style":999},[242],[],{"type":150,"tag":169,"props":23033,"children":23035},{"className":23034},[248],[23036],{"type":155,"value":433},{"type":150,"tag":169,"props":23038,"children":23040},{"className":23039},[248,489],[23041],{"type":155,"value":926},{"type":150,"tag":169,"props":23043,"children":23045},{"className":23044},[1004],[23046],{"type":155,"value":972},{"type":150,"tag":169,"props":23048,"children":23050},{"className":23049},[248],[23051,23056],{"type":150,"tag":169,"props":23052,"children":23054},{"className":23053},[248,489],[23055],{"type":155,"value":759},{"type":150,"tag":169,"props":23057,"children":23059},{"className":23058},[495],[23060],{"type":150,"tag":169,"props":23061,"children":23063},{"className":23062},[253,254],[23064,23093],{"type":150,"tag":169,"props":23065,"children":23067},{"className":23066},[259],[23068,23088],{"type":150,"tag":169,"props":23069,"children":23071},{"className":23070,"style":5039},[264],[23072],{"type":150,"tag":169,"props":23073,"children":23074},{"style":512},[23075,23079],{"type":150,"tag":169,"props":23076,"children":23078},{"className":23077,"style":517},[274],[],{"type":150,"tag":169,"props":23080,"children":23082},{"className":23081},[522,523,524,525],[23083],{"type":150,"tag":169,"props":23084,"children":23086},{"className":23085},[248,489,525],[23087],{"type":155,"value":10046},{"type":150,"tag":169,"props":23089,"children":23091},{"className":23090},[355],[23092],{"type":155,"value":358},{"type":150,"tag":169,"props":23094,"children":23096},{"className":23095},[259],[23097],{"type":150,"tag":169,"props":23098,"children":23100},{"className":23099,"style":544},[264],[23101],{"type":150,"tag":169,"props":23102,"children":23103},{},[],{"type":150,"tag":169,"props":23105,"children":23107},{"className":23106,"style":5167},[558],[],{"type":150,"tag":169,"props":23109,"children":23111},{"className":23110},[5172],[23112],{"type":155,"value":5136},{"type":150,"tag":169,"props":23114,"children":23116},{"className":23115,"style":5167},[558],[],{"type":150,"tag":169,"props":23118,"children":23120},{"className":23119},[237],[23121,23125,23182,23187,23192,23196,23201],{"type":150,"tag":169,"props":23122,"children":23124},{"className":23123,"style":999},[242],[],{"type":150,"tag":169,"props":23126,"children":23128},{"className":23127},[248],[23129,23134],{"type":150,"tag":169,"props":23130,"children":23132},{"className":23131},[248,489],[23133],{"type":155,"value":759},{"type":150,"tag":169,"props":23135,"children":23137},{"className":23136},[495],[23138],{"type":150,"tag":169,"props":23139,"children":23141},{"className":23140},[253,254],[23142,23171],{"type":150,"tag":169,"props":23143,"children":23145},{"className":23144},[259],[23146,23166],{"type":150,"tag":169,"props":23147,"children":23149},{"className":23148,"style":5039},[264],[23150],{"type":150,"tag":169,"props":23151,"children":23152},{"style":512},[23153,23157],{"type":150,"tag":169,"props":23154,"children":23156},{"className":23155,"style":517},[274],[],{"type":150,"tag":169,"props":23158,"children":23160},{"className":23159},[522,523,524,525],[23161],{"type":150,"tag":169,"props":23162,"children":23164},{"className":23163,"style":6968},[248,489,525],[23165],{"type":155,"value":20607},{"type":150,"tag":169,"props":23167,"children":23169},{"className":23168},[355],[23170],{"type":155,"value":358},{"type":150,"tag":169,"props":23172,"children":23174},{"className":23173},[259],[23175],{"type":150,"tag":169,"props":23176,"children":23178},{"className":23177,"style":544},[264],[23179],{"type":150,"tag":169,"props":23180,"children":23181},{},[],{"type":150,"tag":169,"props":23183,"children":23185},{"className":23184},[1015],[23186],{"type":155,"value":982},{"type":150,"tag":169,"props":23188,"children":23190},{"className":23189,"style":5780},[248,489],[23191],{"type":155,"value":5591},{"type":150,"tag":169,"props":23193,"children":23195},{"className":23194,"style":5167},[558],[],{"type":150,"tag":169,"props":23197,"children":23199},{"className":23198},[5172],[23200],{"type":155,"value":5136},{"type":150,"tag":169,"props":23202,"children":23204},{"className":23203,"style":5167},[558],[],{"type":150,"tag":169,"props":23206,"children":23208},{"className":23207},[237],[23209,23213,23218,23223,23280],{"type":150,"tag":169,"props":23210,"children":23212},{"className":23211,"style":4935},[242],[],{"type":150,"tag":169,"props":23214,"children":23216},{"className":23215},[248],[23217],{"type":155,"value":433},{"type":150,"tag":169,"props":23219,"children":23221},{"className":23220},[248,489],[23222],{"type":155,"value":926},{"type":150,"tag":169,"props":23224,"children":23226},{"className":23225},[248],[23227,23232],{"type":150,"tag":169,"props":23228,"children":23230},{"className":23229},[248,489],[23231],{"type":155,"value":759},{"type":150,"tag":169,"props":23233,"children":23235},{"className":23234},[495],[23236],{"type":150,"tag":169,"props":23237,"children":23239},{"className":23238},[253,254],[23240,23269],{"type":150,"tag":169,"props":23241,"children":23243},{"className":23242},[259],[23244,23264],{"type":150,"tag":169,"props":23245,"children":23247},{"className":23246,"style":5039},[264],[23248],{"type":150,"tag":169,"props":23249,"children":23250},{"style":512},[23251,23255],{"type":150,"tag":169,"props":23252,"children":23254},{"className":23253,"style":517},[274],[],{"type":150,"tag":169,"props":23256,"children":23258},{"className":23257},[522,523,524,525],[23259],{"type":150,"tag":169,"props":23260,"children":23262},{"className":23261},[248,489,525],[23263],{"type":155,"value":10046},{"type":150,"tag":169,"props":23265,"children":23267},{"className":23266},[355],[23268],{"type":155,"value":358},{"type":150,"tag":169,"props":23270,"children":23272},{"className":23271},[259],[23273],{"type":150,"tag":169,"props":23274,"children":23276},{"className":23275,"style":544},[264],[23277],{"type":150,"tag":169,"props":23278,"children":23279},{},[],{"type":150,"tag":169,"props":23281,"children":23283},{"className":23282,"style":5780},[248,489],[23284],{"type":155,"value":5591},{"type":150,"tag":151,"props":23286,"children":23287},{},[23288],{"type":155,"value":23289},"代进数字：",{"type":150,"tag":22368,"props":23291,"children":23292},{},[23293,23315],{"type":150,"tag":22372,"props":23294,"children":23295},{},[23296],{"type":150,"tag":22376,"props":23297,"children":23298},{},[23299,23302,23306,23310],{"type":150,"tag":22380,"props":23300,"children":23301},{},[],{"type":150,"tag":22380,"props":23303,"children":23304},{},[23305],{"type":155,"value":22405},{"type":150,"tag":22380,"props":23307,"children":23308},{},[23309],{"type":155,"value":13},{"type":150,"tag":22380,"props":23311,"children":23312},{},[23313],{"type":155,"value":23314},"比值",{"type":150,"tag":22394,"props":23316,"children":23317},{},[23318,23449],{"type":150,"tag":22376,"props":23319,"children":23320},{},[23321,23326,23384,23441],{"type":150,"tag":22401,"props":23322,"children":23323},{},[23324],{"type":155,"value":23325},"KV 读取 \u002F 层 \u002F token",{"type":150,"tag":22401,"props":23327,"children":23328},{},[23329,23382],{"type":150,"tag":169,"props":23330,"children":23332},{"className":23331},[177],[23333,23360],{"type":150,"tag":169,"props":23334,"children":23336},{"className":23335},[182],[23337],{"type":150,"tag":185,"props":23338,"children":23339},{"xmlns":187},[23340],{"type":150,"tag":191,"props":23341,"children":23342},{},[23343,23355],{"type":150,"tag":195,"props":23344,"children":23345},{},[23346,23351],{"type":150,"tag":411,"props":23347,"children":23348},{},[23349],{"type":155,"value":23350},"65536",{"type":150,"tag":405,"props":23352,"children":23353},{},[23354],{"type":155,"value":5591},{"type":150,"tag":223,"props":23356,"children":23357},{"encoding":225},[23358],{"type":155,"value":23359},"65536N",{"type":150,"tag":169,"props":23361,"children":23363},{"className":23362,"ariaHidden":214},[232],[23364],{"type":150,"tag":169,"props":23365,"children":23367},{"className":23366},[237],[23368,23372,23377],{"type":150,"tag":169,"props":23369,"children":23371},{"className":23370,"style":5854},[242],[],{"type":150,"tag":169,"props":23373,"children":23375},{"className":23374},[248],[23376],{"type":155,"value":23350},{"type":150,"tag":169,"props":23378,"children":23380},{"className":23379,"style":5780},[248,489],[23381],{"type":155,"value":5591},{"type":155,"value":23383}," B",{"type":150,"tag":22401,"props":23385,"children":23386},{},[23387,23440],{"type":150,"tag":169,"props":23388,"children":23390},{"className":23389},[177],[23391,23418],{"type":150,"tag":169,"props":23392,"children":23394},{"className":23393},[182],[23395],{"type":150,"tag":185,"props":23396,"children":23397},{"xmlns":187},[23398],{"type":150,"tag":191,"props":23399,"children":23400},{},[23401,23413],{"type":150,"tag":195,"props":23402,"children":23403},{},[23404,23409],{"type":150,"tag":411,"props":23405,"children":23406},{},[23407],{"type":155,"value":23408},"1152",{"type":150,"tag":405,"props":23410,"children":23411},{},[23412],{"type":155,"value":5591},{"type":150,"tag":223,"props":23414,"children":23415},{"encoding":225},[23416],{"type":155,"value":23417},"1152N",{"type":150,"tag":169,"props":23419,"children":23421},{"className":23420,"ariaHidden":214},[232],[23422],{"type":150,"tag":169,"props":23423,"children":23425},{"className":23424},[237],[23426,23430,23435],{"type":150,"tag":169,"props":23427,"children":23429},{"className":23428,"style":5854},[242],[],{"type":150,"tag":169,"props":23431,"children":23433},{"className":23432},[248],[23434],{"type":155,"value":23408},{"type":150,"tag":169,"props":23436,"children":23438},{"className":23437,"style":5780},[248,489],[23439],{"type":155,"value":5591},{"type":155,"value":23383},{"type":150,"tag":22401,"props":23442,"children":23443},{},[23444],{"type":150,"tag":158,"props":23445,"children":23446},{},[23447],{"type":155,"value":23448},"1 : 56.9",{"type":150,"tag":22376,"props":23450,"children":23451},{},[23452,23457,23511,23567],{"type":150,"tag":22401,"props":23453,"children":23454},{},[23455],{"type":155,"value":23456},"attention FLOPs \u002F 层 \u002F token",{"type":150,"tag":22401,"props":23458,"children":23459},{},[23460],{"type":150,"tag":169,"props":23461,"children":23463},{"className":23462},[177],[23464,23489],{"type":150,"tag":169,"props":23465,"children":23467},{"className":23466},[182],[23468],{"type":150,"tag":185,"props":23469,"children":23470},{"xmlns":187},[23471],{"type":150,"tag":191,"props":23472,"children":23473},{},[23474,23485],{"type":150,"tag":195,"props":23475,"children":23476},{},[23477,23481],{"type":150,"tag":411,"props":23478,"children":23479},{},[23480],{"type":155,"value":23350},{"type":150,"tag":405,"props":23482,"children":23483},{},[23484],{"type":155,"value":5591},{"type":150,"tag":223,"props":23486,"children":23487},{"encoding":225},[23488],{"type":155,"value":23359},{"type":150,"tag":169,"props":23490,"children":23492},{"className":23491,"ariaHidden":214},[232],[23493],{"type":150,"tag":169,"props":23494,"children":23496},{"className":23495},[237],[23497,23501,23506],{"type":150,"tag":169,"props":23498,"children":23500},{"className":23499,"style":5854},[242],[],{"type":150,"tag":169,"props":23502,"children":23504},{"className":23503},[248],[23505],{"type":155,"value":23350},{"type":150,"tag":169,"props":23507,"children":23509},{"className":23508,"style":5780},[248,489],[23510],{"type":155,"value":5591},{"type":150,"tag":22401,"props":23512,"children":23513},{},[23514],{"type":150,"tag":169,"props":23515,"children":23517},{"className":23516},[177],[23518,23545],{"type":150,"tag":169,"props":23519,"children":23521},{"className":23520},[182],[23522],{"type":150,"tag":185,"props":23523,"children":23524},{"xmlns":187},[23525],{"type":150,"tag":191,"props":23526,"children":23527},{},[23528,23540],{"type":150,"tag":195,"props":23529,"children":23530},{},[23531,23536],{"type":150,"tag":411,"props":23532,"children":23533},{},[23534],{"type":155,"value":23535},"278528",{"type":150,"tag":405,"props":23537,"children":23538},{},[23539],{"type":155,"value":5591},{"type":150,"tag":223,"props":23541,"children":23542},{"encoding":225},[23543],{"type":155,"value":23544},"278528N",{"type":150,"tag":169,"props":23546,"children":23548},{"className":23547,"ariaHidden":214},[232],[23549],{"type":150,"tag":169,"props":23550,"children":23552},{"className":23551},[237],[23553,23557,23562],{"type":150,"tag":169,"props":23554,"children":23556},{"className":23555,"style":5854},[242],[],{"type":150,"tag":169,"props":23558,"children":23560},{"className":23559},[248],[23561],{"type":155,"value":23535},{"type":150,"tag":169,"props":23563,"children":23565},{"className":23564,"style":5780},[248,489],[23566],{"type":155,"value":5591},{"type":150,"tag":22401,"props":23568,"children":23569},{},[23570],{"type":150,"tag":158,"props":23571,"children":23572},{},[23573],{"type":155,"value":23574},"4.25 : 1",{"type":150,"tag":151,"props":23576,"children":23577},{},[23578],{"type":155,"value":23579},"用 roofline 分析一下。H100 SXM 的 BF16 算力约 989 TFLOPS，HBM3 带宽 3.35 TB\u002Fs，拐点在：",{"type":150,"tag":169,"props":23581,"children":23583},{"className":23582},[172],[23584],{"type":150,"tag":169,"props":23585,"children":23587},{"className":23586},[177],[23588,23649],{"type":150,"tag":169,"props":23589,"children":23591},{"className":23590},[182],[23592],{"type":150,"tag":185,"props":23593,"children":23594},{"xmlns":187,"display":188},[23595],{"type":150,"tag":191,"props":23596,"children":23597},{},[23598,23644],{"type":150,"tag":195,"props":23599,"children":23600},{},[23601,23630,23634,23639],{"type":150,"tag":1952,"props":23602,"children":23603},{},[23604,23617],{"type":150,"tag":195,"props":23605,"children":23606},{},[23607,23612],{"type":150,"tag":411,"props":23608,"children":23609},{},[23610],{"type":155,"value":23611},"989",{"type":150,"tag":217,"props":23613,"children":23614},{},[23615],{"type":155,"value":23616}," TFLOPS",{"type":150,"tag":195,"props":23618,"children":23619},{},[23620,23625],{"type":150,"tag":411,"props":23621,"children":23622},{},[23623],{"type":155,"value":23624},"3.35",{"type":150,"tag":217,"props":23626,"children":23627},{},[23628],{"type":155,"value":23629}," TB\u002Fs",{"type":150,"tag":417,"props":23631,"children":23632},{},[23633],{"type":155,"value":6050},{"type":150,"tag":411,"props":23635,"children":23636},{},[23637],{"type":155,"value":23638},"295",{"type":150,"tag":217,"props":23640,"children":23641},{},[23642],{"type":155,"value":23643}," FLOP\u002Fbyte",{"type":150,"tag":223,"props":23645,"children":23646},{"encoding":225},[23647],{"type":155,"value":23648},"\\frac{989\\ \\text{TFLOPS}}{3.35\\ \\text{TB\u002Fs}} \\approx 295\\ \\text{FLOP\u002Fbyte}",{"type":150,"tag":169,"props":23650,"children":23652},{"className":23651,"ariaHidden":214},[232],[23653,23794],{"type":150,"tag":169,"props":23654,"children":23656},{"className":23655},[237],[23657,23662,23781,23785,23790],{"type":150,"tag":169,"props":23658,"children":23661},{"className":23659,"style":23660},[242],"height:2.2963em;vertical-align:-0.936em;",[],{"type":150,"tag":169,"props":23663,"children":23665},{"className":23664},[248],[23666,23670,23777],{"type":150,"tag":169,"props":23667,"children":23669},{"className":23668},[1004,2660],[],{"type":150,"tag":169,"props":23671,"children":23673},{"className":23672},[1952],[23674],{"type":150,"tag":169,"props":23675,"children":23677},{"className":23676},[253,254],[23678,23765],{"type":150,"tag":169,"props":23679,"children":23681},{"className":23680},[259],[23682,23760],{"type":150,"tag":169,"props":23683,"children":23686},{"className":23684,"style":23685},[264],"height:1.3603em;",[23687,23718,23729],{"type":150,"tag":169,"props":23688,"children":23689},{"style":8422},[23690,23694],{"type":150,"tag":169,"props":23691,"children":23693},{"className":23692,"style":8427},[274],[],{"type":150,"tag":169,"props":23695,"children":23697},{"className":23696},[248],[23698,23703,23708],{"type":150,"tag":169,"props":23699,"children":23701},{"className":23700},[248],[23702],{"type":155,"value":23624},{"type":150,"tag":169,"props":23704,"children":23706},{"className":23705},[558],[23707],{"type":155,"value":8763},{"type":150,"tag":169,"props":23709,"children":23711},{"className":23710},[248,155],[23712],{"type":150,"tag":169,"props":23713,"children":23715},{"className":23714},[248],[23716],{"type":155,"value":23717},"TB\u002Fs",{"type":150,"tag":169,"props":23719,"children":23720},{"style":8440},[23721,23725],{"type":150,"tag":169,"props":23722,"children":23724},{"className":23723,"style":8427},[274],[],{"type":150,"tag":169,"props":23726,"children":23728},{"className":23727,"style":2982},[2981],[],{"type":150,"tag":169,"props":23730,"children":23731},{"style":8452},[23732,23736],{"type":150,"tag":169,"props":23733,"children":23735},{"className":23734,"style":8427},[274],[],{"type":150,"tag":169,"props":23737,"children":23739},{"className":23738},[248],[23740,23745,23750],{"type":150,"tag":169,"props":23741,"children":23743},{"className":23742},[248],[23744],{"type":155,"value":23611},{"type":150,"tag":169,"props":23746,"children":23748},{"className":23747},[558],[23749],{"type":155,"value":8763},{"type":150,"tag":169,"props":23751,"children":23753},{"className":23752},[248,155],[23754],{"type":150,"tag":169,"props":23755,"children":23757},{"className":23756},[248],[23758],{"type":155,"value":23759},"TFLOPS",{"type":150,"tag":169,"props":23761,"children":23763},{"className":23762},[355],[23764],{"type":155,"value":358},{"type":150,"tag":169,"props":23766,"children":23768},{"className":23767},[259],[23769],{"type":150,"tag":169,"props":23770,"children":23773},{"className":23771,"style":23772},[264],"height:0.936em;",[23774],{"type":150,"tag":169,"props":23775,"children":23776},{},[],{"type":150,"tag":169,"props":23778,"children":23780},{"className":23779},[1015,2660],[],{"type":150,"tag":169,"props":23782,"children":23784},{"className":23783,"style":839},[558],[],{"type":150,"tag":169,"props":23786,"children":23788},{"className":23787},[844],[23789],{"type":155,"value":6050},{"type":150,"tag":169,"props":23791,"children":23793},{"className":23792,"style":839},[558],[],{"type":150,"tag":169,"props":23795,"children":23797},{"className":23796},[237],[23798,23802,23807,23812],{"type":150,"tag":169,"props":23799,"children":23801},{"className":23800,"style":999},[242],[],{"type":150,"tag":169,"props":23803,"children":23805},{"className":23804},[248],[23806],{"type":155,"value":23638},{"type":150,"tag":169,"props":23808,"children":23810},{"className":23809},[558],[23811],{"type":155,"value":8763},{"type":150,"tag":169,"props":23813,"children":23815},{"className":23814},[248,155],[23816],{"type":150,"tag":169,"props":23817,"children":23819},{"className":23818},[248],[23820],{"type":155,"value":23821},"FLOP\u002Fbyte",{"type":150,"tag":151,"props":23823,"children":23824},{},[23825],{"type":155,"value":23826},"算术强度低于 295 就是 memory bound，高于才是 compute bound。",{"type":150,"tag":151,"props":23828,"children":23829},{},[23830],{"type":155,"value":23831},"再看 MLA decode 这一侧。别忘了每 token 还要把权重读一遍。",{"type":150,"tag":23833,"props":23834,"children":23835},"ul",{},[23836,23884,23965,24397,24881],{"type":150,"tag":23837,"props":23838,"children":23839},"li",{},[23840,23882],{"type":150,"tag":169,"props":23841,"children":23843},{"className":23842},[177],[23844,23865],{"type":150,"tag":169,"props":23845,"children":23847},{"className":23846},[182],[23848],{"type":150,"tag":185,"props":23849,"children":23850},{"xmlns":187},[23851],{"type":150,"tag":191,"props":23852,"children":23853},{},[23854,23861],{"type":150,"tag":195,"props":23855,"children":23856},{},[23857],{"type":150,"tag":405,"props":23858,"children":23859},{},[23860],{"type":155,"value":5591},{"type":150,"tag":223,"props":23862,"children":23863},{"encoding":225},[23864],{"type":155,"value":5591},{"type":150,"tag":169,"props":23866,"children":23868},{"className":23867,"ariaHidden":214},[232],[23869],{"type":150,"tag":169,"props":23870,"children":23872},{"className":23871},[237],[23873,23877],{"type":150,"tag":169,"props":23874,"children":23876},{"className":23875,"style":5854},[242],[],{"type":150,"tag":169,"props":23878,"children":23880},{"className":23879,"style":5780},[248,489],[23881],{"type":155,"value":5591},{"type":155,"value":23883},"：当前 context 长度，也就是 decode 到这一步时要看多少个历史 token",{"type":150,"tag":23837,"props":23885,"children":23886},{},[23887,23963],{"type":150,"tag":169,"props":23888,"children":23890},{"className":23889},[177],[23891,23920],{"type":150,"tag":169,"props":23892,"children":23894},{"className":23893},[182],[23895],{"type":150,"tag":185,"props":23896,"children":23897},{"xmlns":187},[23898],{"type":150,"tag":191,"props":23899,"children":23900},{},[23901,23916],{"type":150,"tag":195,"props":23902,"children":23903},{},[23904,23908,23912],{"type":150,"tag":405,"props":23905,"children":23906},{},[23907],{"type":155,"value":5559},{"type":150,"tag":417,"props":23909,"children":23910},{},[23911],{"type":155,"value":1045},{"type":150,"tag":411,"props":23913,"children":23914},{},[23915],{"type":155,"value":21701},{"type":150,"tag":223,"props":23917,"children":23918},{"encoding":225},[23919],{"type":155,"value":21706},{"type":150,"tag":169,"props":23921,"children":23923},{"className":23922,"ariaHidden":214},[232],[23924,23950],{"type":150,"tag":169,"props":23925,"children":23927},{"className":23926},[237],[23928,23932,23937,23941,23946],{"type":150,"tag":169,"props":23929,"children":23931},{"className":23930,"style":5854},[242],[],{"type":150,"tag":169,"props":23933,"children":23935},{"className":23934},[248,489],[23936],{"type":155,"value":5559},{"type":150,"tag":169,"props":23938,"children":23940},{"className":23939,"style":839},[558],[],{"type":150,"tag":169,"props":23942,"children":23944},{"className":23943},[844],[23945],{"type":155,"value":1045},{"type":150,"tag":169,"props":23947,"children":23949},{"className":23948,"style":839},[558],[],{"type":150,"tag":169,"props":23951,"children":23953},{"className":23952},[237],[23954,23958],{"type":150,"tag":169,"props":23955,"children":23957},{"className":23956,"style":5186},[242],[],{"type":150,"tag":169,"props":23959,"children":23961},{"className":23960},[248],[23962],{"type":155,"value":21701},{"type":155,"value":23964},"：层数",{"type":150,"tag":23837,"props":23966,"children":23967},{},[23968,24395],{"type":150,"tag":169,"props":23969,"children":23971},{"className":23970},[177],[23972,24069],{"type":150,"tag":169,"props":23973,"children":23975},{"className":23974},[182],[23976],{"type":150,"tag":185,"props":23977,"children":23978},{"xmlns":187},[23979],{"type":150,"tag":191,"props":23980,"children":23981},{},[23982,24064],{"type":150,"tag":195,"props":23983,"children":23984},{},[23985,23997,24001,24005,24016,24020,24031,24035,24039,24043,24048,24052,24056,24060],{"type":150,"tag":401,"props":23986,"children":23987},{},[23988,23992],{"type":150,"tag":405,"props":23989,"children":23990},{},[23991],{"type":155,"value":10046},{"type":150,"tag":217,"props":23993,"children":23994},{},[23995],{"type":155,"value":23996},"kv",{"type":150,"tag":417,"props":23998,"children":23999},{},[24000],{"type":155,"value":1045},{"type":150,"tag":417,"props":24002,"children":24003},{"stretchy":207},[24004],{"type":155,"value":972},{"type":150,"tag":401,"props":24006,"children":24007},{},[24008,24012],{"type":150,"tag":405,"props":24009,"children":24010},{},[24011],{"type":155,"value":759},{"type":150,"tag":405,"props":24013,"children":24014},{},[24015],{"type":155,"value":10046},{"type":150,"tag":417,"props":24017,"children":24018},{},[24019],{"type":155,"value":5136},{"type":150,"tag":401,"props":24021,"children":24022},{},[24023,24027],{"type":150,"tag":405,"props":24024,"children":24025},{},[24026],{"type":155,"value":759},{"type":150,"tag":405,"props":24028,"children":24029},{},[24030],{"type":155,"value":20607},{"type":150,"tag":417,"props":24032,"children":24033},{"stretchy":207},[24034],{"type":155,"value":982},{"type":150,"tag":405,"props":24036,"children":24037},{},[24038],{"type":155,"value":22264},{"type":150,"tag":417,"props":24040,"children":24041},{},[24042],{"type":155,"value":1045},{"type":150,"tag":411,"props":24044,"children":24045},{},[24046],{"type":155,"value":24047},"576",{"type":150,"tag":417,"props":24049,"children":24050},{},[24051],{"type":155,"value":5554},{"type":150,"tag":411,"props":24053,"children":24054},{},[24055],{"type":155,"value":433},{"type":150,"tag":417,"props":24057,"children":24058},{},[24059],{"type":155,"value":1045},{"type":150,"tag":411,"props":24061,"children":24062},{},[24063],{"type":155,"value":23408},{"type":150,"tag":223,"props":24065,"children":24066},{"encoding":225},[24067],{"type":155,"value":24068},"c_{\\text{kv}} = (d_c + d_r)b = 576 \\times 2 = 1152",{"type":150,"tag":169,"props":24070,"children":24072},{"className":24071,"ariaHidden":214},[232],[24073,24159,24242,24330,24356,24382],{"type":150,"tag":169,"props":24074,"children":24076},{"className":24075},[237],[24077,24081,24146,24150,24155],{"type":150,"tag":169,"props":24078,"children":24080},{"className":24079,"style":1343},[242],[],{"type":150,"tag":169,"props":24082,"children":24084},{"className":24083},[248],[24085,24090],{"type":150,"tag":169,"props":24086,"children":24088},{"className":24087},[248,489],[24089],{"type":155,"value":10046},{"type":150,"tag":169,"props":24091,"children":24093},{"className":24092},[495],[24094],{"type":150,"tag":169,"props":24095,"children":24097},{"className":24096},[253,254],[24098,24135],{"type":150,"tag":169,"props":24099,"children":24101},{"className":24100},[259],[24102,24130],{"type":150,"tag":169,"props":24103,"children":24105},{"className":24104,"style":674},[264],[24106],{"type":150,"tag":169,"props":24107,"children":24108},{"style":512},[24109,24113],{"type":150,"tag":169,"props":24110,"children":24112},{"className":24111,"style":517},[274],[],{"type":150,"tag":169,"props":24114,"children":24116},{"className":24115},[522,523,524,525],[24117],{"type":150,"tag":169,"props":24118,"children":24120},{"className":24119},[248,525],[24121],{"type":150,"tag":169,"props":24122,"children":24124},{"className":24123},[248,155,525],[24125],{"type":150,"tag":169,"props":24126,"children":24128},{"className":24127},[248,525],[24129],{"type":155,"value":23996},{"type":150,"tag":169,"props":24131,"children":24133},{"className":24132},[355],[24134],{"type":155,"value":358},{"type":150,"tag":169,"props":24136,"children":24138},{"className":24137},[259],[24139],{"type":150,"tag":169,"props":24140,"children":24142},{"className":24141,"style":544},[264],[24143],{"type":150,"tag":169,"props":24144,"children":24145},{},[],{"type":150,"tag":169,"props":24147,"children":24149},{"className":24148,"style":839},[558],[],{"type":150,"tag":169,"props":24151,"children":24153},{"className":24152},[844],[24154],{"type":155,"value":1045},{"type":150,"tag":169,"props":24156,"children":24158},{"className":24157,"style":839},[558],[],{"type":150,"tag":169,"props":24160,"children":24162},{"className":24161},[237],[24163,24167,24172,24229,24233,24238],{"type":150,"tag":169,"props":24164,"children":24166},{"className":24165,"style":999},[242],[],{"type":150,"tag":169,"props":24168,"children":24170},{"className":24169},[1004],[24171],{"type":155,"value":972},{"type":150,"tag":169,"props":24173,"children":24175},{"className":24174},[248],[24176,24181],{"type":150,"tag":169,"props":24177,"children":24179},{"className":24178},[248,489],[24180],{"type":155,"value":759},{"type":150,"tag":169,"props":24182,"children":24184},{"className":24183},[495],[24185],{"type":150,"tag":169,"props":24186,"children":24188},{"className":24187},[253,254],[24189,24218],{"type":150,"tag":169,"props":24190,"children":24192},{"className":24191},[259],[24193,24213],{"type":150,"tag":169,"props":24194,"children":24196},{"className":24195,"style":5039},[264],[24197],{"type":150,"tag":169,"props":24198,"children":24199},{"style":512},[24200,24204],{"type":150,"tag":169,"props":24201,"children":24203},{"className":24202,"style":517},[274],[],{"type":150,"tag":169,"props":24205,"children":24207},{"className":24206},[522,523,524,525],[24208],{"type":150,"tag":169,"props":24209,"children":24211},{"className":24210},[248,489,525],[24212],{"type":155,"value":10046},{"type":150,"tag":169,"props":24214,"children":24216},{"className":24215},[355],[24217],{"type":155,"value":358},{"type":150,"tag":169,"props":24219,"children":24221},{"className":24220},[259],[24222],{"type":150,"tag":169,"props":24223,"children":24225},{"className":24224,"style":544},[264],[24226],{"type":150,"tag":169,"props":24227,"children":24228},{},[],{"type":150,"tag":169,"props":24230,"children":24232},{"className":24231,"style":5167},[558],[],{"type":150,"tag":169,"props":24234,"children":24236},{"className":24235},[5172],[24237],{"type":155,"value":5136},{"type":150,"tag":169,"props":24239,"children":24241},{"className":24240,"style":5167},[558],[],{"type":150,"tag":169,"props":24243,"children":24245},{"className":24244},[237],[24246,24250,24307,24312,24317,24321,24326],{"type":150,"tag":169,"props":24247,"children":24249},{"className":24248,"style":999},[242],[],{"type":150,"tag":169,"props":24251,"children":24253},{"className":24252},[248],[24254,24259],{"type":150,"tag":169,"props":24255,"children":24257},{"className":24256},[248,489],[24258],{"type":155,"value":759},{"type":150,"tag":169,"props":24260,"children":24262},{"className":24261},[495],[24263],{"type":150,"tag":169,"props":24264,"children":24266},{"className":24265},[253,254],[24267,24296],{"type":150,"tag":169,"props":24268,"children":24270},{"className":24269},[259],[24271,24291],{"type":150,"tag":169,"props":24272,"children":24274},{"className":24273,"style":5039},[264],[24275],{"type":150,"tag":169,"props":24276,"children":24277},{"style":512},[24278,24282],{"type":150,"tag":169,"props":24279,"children":24281},{"className":24280,"style":517},[274],[],{"type":150,"tag":169,"props":24283,"children":24285},{"className":24284},[522,523,524,525],[24286],{"type":150,"tag":169,"props":24287,"children":24289},{"className":24288,"style":6968},[248,489,525],[24290],{"type":155,"value":20607},{"type":150,"tag":169,"props":24292,"children":24294},{"className":24293},[355],[24295],{"type":155,"value":358},{"type":150,"tag":169,"props":24297,"children":24299},{"className":24298},[259],[24300],{"type":150,"tag":169,"props":24301,"children":24303},{"className":24302,"style":544},[264],[24304],{"type":150,"tag":169,"props":24305,"children":24306},{},[],{"type":150,"tag":169,"props":24308,"children":24310},{"className":24309},[1015],[24311],{"type":155,"value":982},{"type":150,"tag":169,"props":24313,"children":24315},{"className":24314},[248,489],[24316],{"type":155,"value":22264},{"type":150,"tag":169,"props":24318,"children":24320},{"className":24319,"style":839},[558],[],{"type":150,"tag":169,"props":24322,"children":24324},{"className":24323},[844],[24325],{"type":155,"value":1045},{"type":150,"tag":169,"props":24327,"children":24329},{"className":24328,"style":839},[558],[],{"type":150,"tag":169,"props":24331,"children":24333},{"className":24332},[237],[24334,24338,24343,24347,24352],{"type":150,"tag":169,"props":24335,"children":24337},{"className":24336,"style":5617},[242],[],{"type":150,"tag":169,"props":24339,"children":24341},{"className":24340},[248],[24342],{"type":155,"value":24047},{"type":150,"tag":169,"props":24344,"children":24346},{"className":24345,"style":5167},[558],[],{"type":150,"tag":169,"props":24348,"children":24350},{"className":24349},[5172],[24351],{"type":155,"value":5554},{"type":150,"tag":169,"props":24353,"children":24355},{"className":24354,"style":5167},[558],[],{"type":150,"tag":169,"props":24357,"children":24359},{"className":24358},[237],[24360,24364,24369,24373,24378],{"type":150,"tag":169,"props":24361,"children":24363},{"className":24362,"style":5186},[242],[],{"type":150,"tag":169,"props":24365,"children":24367},{"className":24366},[248],[24368],{"type":155,"value":433},{"type":150,"tag":169,"props":24370,"children":24372},{"className":24371,"style":839},[558],[],{"type":150,"tag":169,"props":24374,"children":24376},{"className":24375},[844],[24377],{"type":155,"value":1045},{"type":150,"tag":169,"props":24379,"children":24381},{"className":24380,"style":839},[558],[],{"type":150,"tag":169,"props":24383,"children":24385},{"className":24384},[237],[24386,24390],{"type":150,"tag":169,"props":24387,"children":24389},{"className":24388,"style":5186},[242],[],{"type":150,"tag":169,"props":24391,"children":24393},{"className":24392},[248],[24394],{"type":155,"value":23408},{"type":155,"value":24396}," 字节：每层、每 token、每多看一个历史位置，要多读多少 KV",{"type":150,"tag":23837,"props":24398,"children":24399},{},[24400,24879],{"type":150,"tag":169,"props":24401,"children":24403},{"className":24402},[177],[24404,24511],{"type":150,"tag":169,"props":24405,"children":24407},{"className":24406},[182],[24408],{"type":150,"tag":185,"props":24409,"children":24410},{"xmlns":187},[24411],{"type":150,"tag":191,"props":24412,"children":24413},{},[24414,24506],{"type":150,"tag":195,"props":24415,"children":24416},{},[24417,24429,24433,24437,24441,24445,24456,24460,24471,24475,24479,24483,24487,24498,24502],{"type":150,"tag":401,"props":24418,"children":24419},{},[24420,24424],{"type":150,"tag":405,"props":24421,"children":24422},{},[24423],{"type":155,"value":10046},{"type":150,"tag":217,"props":24425,"children":24426},{},[24427],{"type":155,"value":24428},"flop",{"type":150,"tag":417,"props":24430,"children":24431},{},[24432],{"type":155,"value":1045},{"type":150,"tag":411,"props":24434,"children":24435},{},[24436],{"type":155,"value":433},{"type":150,"tag":405,"props":24438,"children":24439},{},[24440],{"type":155,"value":926},{"type":150,"tag":417,"props":24442,"children":24443},{"stretchy":207},[24444],{"type":155,"value":972},{"type":150,"tag":401,"props":24446,"children":24447},{},[24448,24452],{"type":150,"tag":405,"props":24449,"children":24450},{},[24451],{"type":155,"value":759},{"type":150,"tag":405,"props":24453,"children":24454},{},[24455],{"type":155,"value":10046},{"type":150,"tag":417,"props":24457,"children":24458},{},[24459],{"type":155,"value":5136},{"type":150,"tag":401,"props":24461,"children":24462},{},[24463,24467],{"type":150,"tag":405,"props":24464,"children":24465},{},[24466],{"type":155,"value":759},{"type":150,"tag":405,"props":24468,"children":24469},{},[24470],{"type":155,"value":20607},{"type":150,"tag":417,"props":24472,"children":24473},{"stretchy":207},[24474],{"type":155,"value":982},{"type":150,"tag":417,"props":24476,"children":24477},{},[24478],{"type":155,"value":5136},{"type":150,"tag":411,"props":24480,"children":24481},{},[24482],{"type":155,"value":433},{"type":150,"tag":405,"props":24484,"children":24485},{},[24486],{"type":155,"value":926},{"type":150,"tag":401,"props":24488,"children":24489},{},[24490,24494],{"type":150,"tag":405,"props":24491,"children":24492},{},[24493],{"type":155,"value":759},{"type":150,"tag":405,"props":24495,"children":24496},{},[24497],{"type":155,"value":10046},{"type":150,"tag":417,"props":24499,"children":24500},{},[24501],{"type":155,"value":1045},{"type":150,"tag":411,"props":24503,"children":24504},{},[24505],{"type":155,"value":23535},{"type":150,"tag":223,"props":24507,"children":24508},{"encoding":225},[24509],{"type":155,"value":24510},"c_{\\text{flop}} = 2h(d_c + d_r) + 2hd_c = 278528",{"type":150,"tag":169,"props":24512,"children":24514},{"className":24513,"ariaHidden":214},[232],[24515,24602,24695,24778,24866],{"type":150,"tag":169,"props":24516,"children":24518},{"className":24517},[237],[24519,24524,24589,24593,24598],{"type":150,"tag":169,"props":24520,"children":24523},{"className":24521,"style":24522},[242],"height:0.7167em;vertical-align:-0.2861em;",[],{"type":150,"tag":169,"props":24525,"children":24527},{"className":24526},[248],[24528,24533],{"type":150,"tag":169,"props":24529,"children":24531},{"className":24530},[248,489],[24532],{"type":155,"value":10046},{"type":150,"tag":169,"props":24534,"children":24536},{"className":24535},[495],[24537],{"type":150,"tag":169,"props":24538,"children":24540},{"className":24539},[253,254],[24541,24578],{"type":150,"tag":169,"props":24542,"children":24544},{"className":24543},[259],[24545,24573],{"type":150,"tag":169,"props":24546,"children":24548},{"className":24547,"style":674},[264],[24549],{"type":150,"tag":169,"props":24550,"children":24551},{"style":512},[24552,24556],{"type":150,"tag":169,"props":24553,"children":24555},{"className":24554,"style":517},[274],[],{"type":150,"tag":169,"props":24557,"children":24559},{"className":24558},[522,523,524,525],[24560],{"type":150,"tag":169,"props":24561,"children":24563},{"className":24562},[248,525],[24564],{"type":150,"tag":169,"props":24565,"children":24567},{"className":24566},[248,155,525],[24568],{"type":150,"tag":169,"props":24569,"children":24571},{"className":24570},[248,525],[24572],{"type":155,"value":24428},{"type":150,"tag":169,"props":24574,"children":24576},{"className":24575},[355],[24577],{"type":155,"value":358},{"type":150,"tag":169,"props":24579,"children":24581},{"className":24580},[259],[24582],{"type":150,"tag":169,"props":24583,"children":24585},{"className":24584,"style":6692},[264],[24586],{"type":150,"tag":169,"props":24587,"children":24588},{},[],{"type":150,"tag":169,"props":24590,"children":24592},{"className":24591,"style":839},[558],[],{"type":150,"tag":169,"props":24594,"children":24596},{"className":24595},[844],[24597],{"type":155,"value":1045},{"type":150,"tag":169,"props":24599,"children":24601},{"className":24600,"style":839},[558],[],{"type":150,"tag":169,"props":24603,"children":24605},{"className":24604},[237],[24606,24610,24615,24620,24625,24682,24686,24691],{"type":150,"tag":169,"props":24607,"children":24609},{"className":24608,"style":999},[242],[],{"type":150,"tag":169,"props":24611,"children":24613},{"className":24612},[248],[24614],{"type":155,"value":433},{"type":150,"tag":169,"props":24616,"children":24618},{"className":24617},[248,489],[24619],{"type":155,"value":926},{"type":150,"tag":169,"props":24621,"children":24623},{"className":24622},[1004],[24624],{"type":155,"value":972},{"type":150,"tag":169,"props":24626,"children":24628},{"className":24627},[248],[24629,24634],{"type":150,"tag":169,"props":24630,"children":24632},{"className":24631},[248,489],[24633],{"type":155,"value":759},{"type":150,"tag":169,"props":24635,"children":24637},{"className":24636},[495],[24638],{"type":150,"tag":169,"props":24639,"children":24641},{"className":24640},[253,254],[24642,24671],{"type":150,"tag":169,"props":24643,"children":24645},{"className":24644},[259],[24646,24666],{"type":150,"tag":169,"props":24647,"children":24649},{"className":24648,"style":5039},[264],[24650],{"type":150,"tag":169,"props":24651,"children":24652},{"style":512},[24653,24657],{"type":150,"tag":169,"props":24654,"children":24656},{"className":24655,"style":517},[274],[],{"type":150,"tag":169,"props":24658,"children":24660},{"className":24659},[522,523,524,525],[24661],{"type":150,"tag":169,"props":24662,"children":24664},{"className":24663},[248,489,525],[24665],{"type":155,"value":10046},{"type":150,"tag":169,"props":24667,"children":24669},{"className":24668},[355],[24670],{"type":155,"value":358},{"type":150,"tag":169,"props":24672,"children":24674},{"className":24673},[259],[24675],{"type":150,"tag":169,"props":24676,"children":24678},{"className":24677,"style":544},[264],[24679],{"type":150,"tag":169,"props":24680,"children":24681},{},[],{"type":150,"tag":169,"props":24683,"children":24685},{"className":24684,"style":5167},[558],[],{"type":150,"tag":169,"props":24687,"children":24689},{"className":24688},[5172],[24690],{"type":155,"value":5136},{"type":150,"tag":169,"props":24692,"children":24694},{"className":24693,"style":5167},[558],[],{"type":150,"tag":169,"props":24696,"children":24698},{"className":24697},[237],[24699,24703,24760,24765,24769,24774],{"type":150,"tag":169,"props":24700,"children":24702},{"className":24701,"style":999},[242],[],{"type":150,"tag":169,"props":24704,"children":24706},{"className":24705},[248],[24707,24712],{"type":150,"tag":169,"props":24708,"children":24710},{"className":24709},[248,489],[24711],{"type":155,"value":759},{"type":150,"tag":169,"props":24713,"children":24715},{"className":24714},[495],[24716],{"type":150,"tag":169,"props":24717,"children":24719},{"className":24718},[253,254],[24720,24749],{"type":150,"tag":169,"props":24721,"children":24723},{"className":24722},[259],[24724,24744],{"type":150,"tag":169,"props":24725,"children":24727},{"className":24726,"style":5039},[264],[24728],{"type":150,"tag":169,"props":24729,"children":24730},{"style":512},[24731,24735],{"type":150,"tag":169,"props":24732,"children":24734},{"className":24733,"style":517},[274],[],{"type":150,"tag":169,"props":24736,"children":24738},{"className":24737},[522,523,524,525],[24739],{"type":150,"tag":169,"props":24740,"children":24742},{"className":24741,"style":6968},[248,489,525],[24743],{"type":155,"value":20607},{"type":150,"tag":169,"props":24745,"children":24747},{"className":24746},[355],[24748],{"type":155,"value":358},{"type":150,"tag":169,"props":24750,"children":24752},{"className":24751},[259],[24753],{"type":150,"tag":169,"props":24754,"children":24756},{"className":24755,"style":544},[264],[24757],{"type":150,"tag":169,"props":24758,"children":24759},{},[],{"type":150,"tag":169,"props":24761,"children":24763},{"className":24762},[1015],[24764],{"type":155,"value":982},{"type":150,"tag":169,"props":24766,"children":24768},{"className":24767,"style":5167},[558],[],{"type":150,"tag":169,"props":24770,"children":24772},{"className":24771},[5172],[24773],{"type":155,"value":5136},{"type":150,"tag":169,"props":24775,"children":24777},{"className":24776,"style":5167},[558],[],{"type":150,"tag":169,"props":24779,"children":24781},{"className":24780},[237],[24782,24786,24791,24796,24853,24857,24862],{"type":150,"tag":169,"props":24783,"children":24785},{"className":24784,"style":4935},[242],[],{"type":150,"tag":169,"props":24787,"children":24789},{"className":24788},[248],[24790],{"type":155,"value":433},{"type":150,"tag":169,"props":24792,"children":24794},{"className":24793},[248,489],[24795],{"type":155,"value":926},{"type":150,"tag":169,"props":24797,"children":24799},{"className":24798},[248],[24800,24805],{"type":150,"tag":169,"props":24801,"children":24803},{"className":24802},[248,489],[24804],{"type":155,"value":759},{"type":150,"tag":169,"props":24806,"children":24808},{"className":24807},[495],[24809],{"type":150,"tag":169,"props":24810,"children":24812},{"className":24811},[253,254],[24813,24842],{"type":150,"tag":169,"props":24814,"children":24816},{"className":24815},[259],[24817,24837],{"type":150,"tag":169,"props":24818,"children":24820},{"className":24819,"style":5039},[264],[24821],{"type":150,"tag":169,"props":24822,"children":24823},{"style":512},[24824,24828],{"type":150,"tag":169,"props":24825,"children":24827},{"className":24826,"style":517},[274],[],{"type":150,"tag":169,"props":24829,"children":24831},{"className":24830},[522,523,524,525],[24832],{"type":150,"tag":169,"props":24833,"children":24835},{"className":24834},[248,489,525],[24836],{"type":155,"value":10046},{"type":150,"tag":169,"props":24838,"children":24840},{"className":24839},[355],[24841],{"type":155,"value":358},{"type":150,"tag":169,"props":24843,"children":24845},{"className":24844},[259],[24846],{"type":150,"tag":169,"props":24847,"children":24849},{"className":24848,"style":544},[264],[24850],{"type":150,"tag":169,"props":24851,"children":24852},{},[],{"type":150,"tag":169,"props":24854,"children":24856},{"className":24855,"style":839},[558],[],{"type":150,"tag":169,"props":24858,"children":24860},{"className":24859},[844],[24861],{"type":155,"value":1045},{"type":150,"tag":169,"props":24863,"children":24865},{"className":24864,"style":839},[558],[],{"type":150,"tag":169,"props":24867,"children":24869},{"className":24868},[237],[24870,24874],{"type":150,"tag":169,"props":24871,"children":24873},{"className":24872,"style":5186},[242],[],{"type":150,"tag":169,"props":24875,"children":24877},{"className":24876},[248],[24878],{"type":155,"value":23535},{"type":155,"value":24880},"：每层、每 token、每多看一个历史位置，要多做多少次浮点运算",{"type":150,"tag":23837,"props":24882,"children":24883},{},[24884,25065,25067,25109],{"type":150,"tag":169,"props":24885,"children":24887},{"className":24886},[177],[24888,24946],{"type":150,"tag":169,"props":24889,"children":24891},{"className":24890},[182],[24892],{"type":150,"tag":185,"props":24893,"children":24894},{"xmlns":187},[24895],{"type":150,"tag":191,"props":24896,"children":24897},{},[24898,24941],{"type":150,"tag":195,"props":24899,"children":24900},{},[24901,24905,24909,24914,24919,24923,24927,24932,24936],{"type":150,"tag":405,"props":24902,"children":24903},{},[24904],{"type":155,"value":3443},{"type":150,"tag":417,"props":24906,"children":24907},{},[24908],{"type":155,"value":1045},{"type":150,"tag":411,"props":24910,"children":24911},{},[24912],{"type":155,"value":24913},"21",{"type":150,"tag":217,"props":24915,"children":24916},{},[24917],{"type":155,"value":24918},"B",{"type":150,"tag":417,"props":24920,"children":24921},{},[24922],{"type":155,"value":5554},{"type":150,"tag":411,"props":24924,"children":24925},{},[24926],{"type":155,"value":433},{"type":150,"tag":217,"props":24928,"children":24929},{},[24930],{"type":155,"value":24931}," 字节",{"type":150,"tag":417,"props":24933,"children":24934},{},[24935],{"type":155,"value":1045},{"type":150,"tag":411,"props":24937,"children":24938},{},[24939],{"type":155,"value":24940},"42",{"type":150,"tag":223,"props":24942,"children":24943},{"encoding":225},[24944],{"type":155,"value":24945},"W = 21\\text{B} \\times 2\\ \\text{字节} = 42",{"type":150,"tag":169,"props":24947,"children":24949},{"className":24948,"ariaHidden":214},[232],[24950,24976,25011,25052],{"type":150,"tag":169,"props":24951,"children":24953},{"className":24952},[237],[24954,24958,24963,24967,24972],{"type":150,"tag":169,"props":24955,"children":24957},{"className":24956,"style":5854},[242],[],{"type":150,"tag":169,"props":24959,"children":24961},{"className":24960,"style":3869},[248,489],[24962],{"type":155,"value":3443},{"type":150,"tag":169,"props":24964,"children":24966},{"className":24965,"style":839},[558],[],{"type":150,"tag":169,"props":24968,"children":24970},{"className":24969},[844],[24971],{"type":155,"value":1045},{"type":150,"tag":169,"props":24973,"children":24975},{"className":24974,"style":839},[558],[],{"type":150,"tag":169,"props":24977,"children":24979},{"className":24978},[237],[24980,24984,24989,24998,25002,25007],{"type":150,"tag":169,"props":24981,"children":24983},{"className":24982,"style":5644},[242],[],{"type":150,"tag":169,"props":24985,"children":24987},{"className":24986},[248],[24988],{"type":155,"value":24913},{"type":150,"tag":169,"props":24990,"children":24992},{"className":24991},[248,155],[24993],{"type":150,"tag":169,"props":24994,"children":24996},{"className":24995},[248],[24997],{"type":155,"value":24918},{"type":150,"tag":169,"props":24999,"children":25001},{"className":25000,"style":5167},[558],[],{"type":150,"tag":169,"props":25003,"children":25005},{"className":25004},[5172],[25006],{"type":155,"value":5554},{"type":150,"tag":169,"props":25008,"children":25010},{"className":25009,"style":5167},[558],[],{"type":150,"tag":169,"props":25012,"children":25014},{"className":25013},[237],[25015,25019,25024,25029,25039,25043,25048],{"type":150,"tag":169,"props":25016,"children":25018},{"className":25017,"style":5854},[242],[],{"type":150,"tag":169,"props":25020,"children":25022},{"className":25021},[248],[25023],{"type":155,"value":433},{"type":150,"tag":169,"props":25025,"children":25027},{"className":25026},[558],[25028],{"type":155,"value":8763},{"type":150,"tag":169,"props":25030,"children":25032},{"className":25031},[248,155],[25033],{"type":150,"tag":169,"props":25034,"children":25036},{"className":25035},[248,297],[25037],{"type":155,"value":25038},"字节",{"type":150,"tag":169,"props":25040,"children":25042},{"className":25041,"style":839},[558],[],{"type":150,"tag":169,"props":25044,"children":25046},{"className":25045},[844],[25047],{"type":155,"value":1045},{"type":150,"tag":169,"props":25049,"children":25051},{"className":25050,"style":839},[558],[],{"type":150,"tag":169,"props":25053,"children":25055},{"className":25054},[237],[25056,25060],{"type":150,"tag":169,"props":25057,"children":25059},{"className":25058,"style":5186},[242],[],{"type":150,"tag":169,"props":25061,"children":25063},{"className":25062},[248],[25064],{"type":155,"value":24940},{"type":155,"value":25066}," GB：每 token 要读的权重字节数（21B 是 activated 参数量），它不随 ",{"type":150,"tag":169,"props":25068,"children":25070},{"className":25069},[177],[25071,25092],{"type":150,"tag":169,"props":25072,"children":25074},{"className":25073},[182],[25075],{"type":150,"tag":185,"props":25076,"children":25077},{"xmlns":187},[25078],{"type":150,"tag":191,"props":25079,"children":25080},{},[25081,25088],{"type":150,"tag":195,"props":25082,"children":25083},{},[25084],{"type":150,"tag":405,"props":25085,"children":25086},{},[25087],{"type":155,"value":5591},{"type":150,"tag":223,"props":25089,"children":25090},{"encoding":225},[25091],{"type":155,"value":5591},{"type":150,"tag":169,"props":25093,"children":25095},{"className":25094,"ariaHidden":214},[232],[25096],{"type":150,"tag":169,"props":25097,"children":25099},{"className":25098},[237],[25100,25104],{"type":150,"tag":169,"props":25101,"children":25103},{"className":25102,"style":5854},[242],[],{"type":150,"tag":169,"props":25105,"children":25107},{"className":25106,"style":5780},[248,489],[25108],{"type":155,"value":5591},{"type":155,"value":25110}," 变",{"type":150,"tag":151,"props":25112,"children":25113},{},[25114],{"type":155,"value":25115},"算术强度就是一个字节要承担多少次浮点运算：",{"type":150,"tag":169,"props":25117,"children":25119},{"className":25118},[172],[25120],{"type":150,"tag":169,"props":25121,"children":25123},{"className":25122},[177],[25124,25248],{"type":150,"tag":169,"props":25125,"children":25127},{"className":25126},[182],[25128],{"type":150,"tag":185,"props":25129,"children":25130},{"xmlns":187,"display":188},[25131],{"type":150,"tag":191,"props":25132,"children":25133},{},[25134,25243],{"type":150,"tag":195,"props":25135,"children":25136},{},[25137,25142,25146,25150,25154,25158],{"type":150,"tag":217,"props":25138,"children":25139},{},[25140],{"type":155,"value":25141},"AI",{"type":150,"tag":417,"props":25143,"children":25144},{"stretchy":207},[25145],{"type":155,"value":972},{"type":150,"tag":405,"props":25147,"children":25148},{},[25149],{"type":155,"value":5591},{"type":150,"tag":417,"props":25151,"children":25152},{"stretchy":207},[25153],{"type":155,"value":982},{"type":150,"tag":417,"props":25155,"children":25156},{},[25157],{"type":155,"value":1045},{"type":150,"tag":1952,"props":25159,"children":25160},{},[25161,25205],{"type":150,"tag":195,"props":25162,"children":25163},{},[25164,25168,25173,25177,25188,25193,25197,25201],{"type":150,"tag":411,"props":25165,"children":25166},{},[25167],{"type":155,"value":433},{"type":150,"tag":405,"props":25169,"children":25170},{},[25171],{"type":155,"value":25172},"P",{"type":150,"tag":417,"props":25174,"children":25175},{},[25176],{"type":155,"value":5136},{"type":150,"tag":401,"props":25178,"children":25179},{},[25180,25184],{"type":150,"tag":405,"props":25181,"children":25182},{},[25183],{"type":155,"value":10046},{"type":150,"tag":217,"props":25185,"children":25186},{},[25187],{"type":155,"value":24428},{"type":150,"tag":417,"props":25189,"children":25190},{},[25191],{"type":155,"value":25192},"⋅",{"type":150,"tag":405,"props":25194,"children":25195},{},[25196],{"type":155,"value":5559},{"type":150,"tag":417,"props":25198,"children":25199},{},[25200],{"type":155,"value":25192},{"type":150,"tag":405,"props":25202,"children":25203},{},[25204],{"type":155,"value":5591},{"type":150,"tag":195,"props":25206,"children":25207},{},[25208,25212,25216,25227,25231,25235,25239],{"type":150,"tag":405,"props":25209,"children":25210},{},[25211],{"type":155,"value":3443},{"type":150,"tag":417,"props":25213,"children":25214},{},[25215],{"type":155,"value":5136},{"type":150,"tag":401,"props":25217,"children":25218},{},[25219,25223],{"type":150,"tag":405,"props":25220,"children":25221},{},[25222],{"type":155,"value":10046},{"type":150,"tag":217,"props":25224,"children":25225},{},[25226],{"type":155,"value":23996},{"type":150,"tag":417,"props":25228,"children":25229},{},[25230],{"type":155,"value":25192},{"type":150,"tag":405,"props":25232,"children":25233},{},[25234],{"type":155,"value":5559},{"type":150,"tag":417,"props":25236,"children":25237},{},[25238],{"type":155,"value":25192},{"type":150,"tag":405,"props":25240,"children":25241},{},[25242],{"type":155,"value":5591},{"type":150,"tag":223,"props":25244,"children":25245},{"encoding":225},[25246],{"type":155,"value":25247},"\\text{AI}(N) = \\frac{2P + c_{\\text{flop}} \\cdot L \\cdot N}{W + c_{\\text{kv}} \\cdot L \\cdot N}",{"type":150,"tag":169,"props":25249,"children":25251},{"className":25250,"ariaHidden":214},[232],[25252,25297],{"type":150,"tag":169,"props":25253,"children":25255},{"className":25254},[237],[25256,25260,25269,25274,25279,25284,25288,25293],{"type":150,"tag":169,"props":25257,"children":25259},{"className":25258,"style":999},[242],[],{"type":150,"tag":169,"props":25261,"children":25263},{"className":25262},[248,155],[25264],{"type":150,"tag":169,"props":25265,"children":25267},{"className":25266},[248],[25268],{"type":155,"value":25141},{"type":150,"tag":169,"props":25270,"children":25272},{"className":25271},[1004],[25273],{"type":155,"value":972},{"type":150,"tag":169,"props":25275,"children":25277},{"className":25276,"style":5780},[248,489],[25278],{"type":155,"value":5591},{"type":150,"tag":169,"props":25280,"children":25282},{"className":25281},[1015],[25283],{"type":155,"value":982},{"type":150,"tag":169,"props":25285,"children":25287},{"className":25286,"style":839},[558],[],{"type":150,"tag":169,"props":25289,"children":25291},{"className":25290},[844],[25292],{"type":155,"value":1045},{"type":150,"tag":169,"props":25294,"children":25296},{"className":25295,"style":839},[558],[],{"type":150,"tag":169,"props":25298,"children":25300},{"className":25299},[237],[25301,25306],{"type":150,"tag":169,"props":25302,"children":25305},{"className":25303,"style":25304},[242],"height:2.1963em;vertical-align:-0.836em;",[],{"type":150,"tag":169,"props":25307,"children":25309},{"className":25308},[248],[25310,25314,25623],{"type":150,"tag":169,"props":25311,"children":25313},{"className":25312},[1004,2660],[],{"type":150,"tag":169,"props":25315,"children":25317},{"className":25316},[1952],[25318],{"type":150,"tag":169,"props":25319,"children":25321},{"className":25320},[253,254],[25322,25611],{"type":150,"tag":169,"props":25323,"children":25325},{"className":25324},[259],[25326,25606],{"type":150,"tag":169,"props":25327,"children":25329},{"className":25328,"style":23685},[264],[25330,25460,25471],{"type":150,"tag":169,"props":25331,"children":25332},{"style":8422},[25333,25337],{"type":150,"tag":169,"props":25334,"children":25336},{"className":25335,"style":8427},[274],[],{"type":150,"tag":169,"props":25338,"children":25340},{"className":25339},[248],[25341,25346,25350,25355,25359,25424,25428,25433,25437,25442,25446,25451,25455],{"type":150,"tag":169,"props":25342,"children":25344},{"className":25343,"style":3869},[248,489],[25345],{"type":155,"value":3443},{"type":150,"tag":169,"props":25347,"children":25349},{"className":25348,"style":5167},[558],[],{"type":150,"tag":169,"props":25351,"children":25353},{"className":25352},[5172],[25354],{"type":155,"value":5136},{"type":150,"tag":169,"props":25356,"children":25358},{"className":25357,"style":5167},[558],[],{"type":150,"tag":169,"props":25360,"children":25362},{"className":25361},[248],[25363,25368],{"type":150,"tag":169,"props":25364,"children":25366},{"className":25365},[248,489],[25367],{"type":155,"value":10046},{"type":150,"tag":169,"props":25369,"children":25371},{"className":25370},[495],[25372],{"type":150,"tag":169,"props":25373,"children":25375},{"className":25374},[253,254],[25376,25413],{"type":150,"tag":169,"props":25377,"children":25379},{"className":25378},[259],[25380,25408],{"type":150,"tag":169,"props":25381,"children":25383},{"className":25382,"style":674},[264],[25384],{"type":150,"tag":169,"props":25385,"children":25386},{"style":512},[25387,25391],{"type":150,"tag":169,"props":25388,"children":25390},{"className":25389,"style":517},[274],[],{"type":150,"tag":169,"props":25392,"children":25394},{"className":25393},[522,523,524,525],[25395],{"type":150,"tag":169,"props":25396,"children":25398},{"className":25397},[248,525],[25399],{"type":150,"tag":169,"props":25400,"children":25402},{"className":25401},[248,155,525],[25403],{"type":150,"tag":169,"props":25404,"children":25406},{"className":25405},[248,525],[25407],{"type":155,"value":23996},{"type":150,"tag":169,"props":25409,"children":25411},{"className":25410},[355],[25412],{"type":155,"value":358},{"type":150,"tag":169,"props":25414,"children":25416},{"className":25415},[259],[25417],{"type":150,"tag":169,"props":25418,"children":25420},{"className":25419,"style":544},[264],[25421],{"type":150,"tag":169,"props":25422,"children":25423},{},[],{"type":150,"tag":169,"props":25425,"children":25427},{"className":25426,"style":5167},[558],[],{"type":150,"tag":169,"props":25429,"children":25431},{"className":25430},[5172],[25432],{"type":155,"value":25192},{"type":150,"tag":169,"props":25434,"children":25436},{"className":25435,"style":5167},[558],[],{"type":150,"tag":169,"props":25438,"children":25440},{"className":25439},[248,489],[25441],{"type":155,"value":5559},{"type":150,"tag":169,"props":25443,"children":25445},{"className":25444,"style":5167},[558],[],{"type":150,"tag":169,"props":25447,"children":25449},{"className":25448},[5172],[25450],{"type":155,"value":25192},{"type":150,"tag":169,"props":25452,"children":25454},{"className":25453,"style":5167},[558],[],{"type":150,"tag":169,"props":25456,"children":25458},{"className":25457,"style":5780},[248,489],[25459],{"type":155,"value":5591},{"type":150,"tag":169,"props":25461,"children":25462},{"style":8440},[25463,25467],{"type":150,"tag":169,"props":25464,"children":25466},{"className":25465,"style":8427},[274],[],{"type":150,"tag":169,"props":25468,"children":25470},{"className":25469,"style":2982},[2981],[],{"type":150,"tag":169,"props":25472,"children":25473},{"style":8452},[25474,25478],{"type":150,"tag":169,"props":25475,"children":25477},{"className":25476,"style":8427},[274],[],{"type":150,"tag":169,"props":25479,"children":25481},{"className":25480},[248],[25482,25487,25492,25496,25501,25505,25570,25574,25579,25583,25588,25592,25597,25601],{"type":150,"tag":169,"props":25483,"children":25485},{"className":25484},[248],[25486],{"type":155,"value":433},{"type":150,"tag":169,"props":25488,"children":25490},{"className":25489,"style":3869},[248,489],[25491],{"type":155,"value":25172},{"type":150,"tag":169,"props":25493,"children":25495},{"className":25494,"style":5167},[558],[],{"type":150,"tag":169,"props":25497,"children":25499},{"className":25498},[5172],[25500],{"type":155,"value":5136},{"type":150,"tag":169,"props":25502,"children":25504},{"className":25503,"style":5167},[558],[],{"type":150,"tag":169,"props":25506,"children":25508},{"className":25507},[248],[25509,25514],{"type":150,"tag":169,"props":25510,"children":25512},{"className":25511},[248,489],[25513],{"type":155,"value":10046},{"type":150,"tag":169,"props":25515,"children":25517},{"className":25516},[495],[25518],{"type":150,"tag":169,"props":25519,"children":25521},{"className":25520},[253,254],[25522,25559],{"type":150,"tag":169,"props":25523,"children":25525},{"className":25524},[259],[25526,25554],{"type":150,"tag":169,"props":25527,"children":25529},{"className":25528,"style":674},[264],[25530],{"type":150,"tag":169,"props":25531,"children":25532},{"style":512},[25533,25537],{"type":150,"tag":169,"props":25534,"children":25536},{"className":25535,"style":517},[274],[],{"type":150,"tag":169,"props":25538,"children":25540},{"className":25539},[522,523,524,525],[25541],{"type":150,"tag":169,"props":25542,"children":25544},{"className":25543},[248,525],[25545],{"type":150,"tag":169,"props":25546,"children":25548},{"className":25547},[248,155,525],[25549],{"type":150,"tag":169,"props":25550,"children":25552},{"className":25551},[248,525],[25553],{"type":155,"value":24428},{"type":150,"tag":169,"props":25555,"children":25557},{"className":25556},[355],[25558],{"type":155,"value":358},{"type":150,"tag":169,"props":25560,"children":25562},{"className":25561},[259],[25563],{"type":150,"tag":169,"props":25564,"children":25566},{"className":25565,"style":6692},[264],[25567],{"type":150,"tag":169,"props":25568,"children":25569},{},[],{"type":150,"tag":169,"props":25571,"children":25573},{"className":25572,"style":5167},[558],[],{"type":150,"tag":169,"props":25575,"children":25577},{"className":25576},[5172],[25578],{"type":155,"value":25192},{"type":150,"tag":169,"props":25580,"children":25582},{"className":25581,"style":5167},[558],[],{"type":150,"tag":169,"props":25584,"children":25586},{"className":25585},[248,489],[25587],{"type":155,"value":5559},{"type":150,"tag":169,"props":25589,"children":25591},{"className":25590,"style":5167},[558],[],{"type":150,"tag":169,"props":25593,"children":25595},{"className":25594},[5172],[25596],{"type":155,"value":25192},{"type":150,"tag":169,"props":25598,"children":25600},{"className":25599,"style":5167},[558],[],{"type":150,"tag":169,"props":25602,"children":25604},{"className":25603,"style":5780},[248,489],[25605],{"type":155,"value":5591},{"type":150,"tag":169,"props":25607,"children":25609},{"className":25608},[355],[25610],{"type":155,"value":358},{"type":150,"tag":169,"props":25612,"children":25614},{"className":25613},[259],[25615],{"type":150,"tag":169,"props":25616,"children":25619},{"className":25617,"style":25618},[264],"height:0.836em;",[25620],{"type":150,"tag":169,"props":25621,"children":25622},{},[],{"type":150,"tag":169,"props":25624,"children":25626},{"className":25625},[1015,2660],[],{"type":150,"tag":151,"props":25628,"children":25629},{},[25630,25632,25684,25686,25728],{"type":155,"value":25631},"分子是每个 token 要做的浮点次数：权重矩阵乘 ",{"type":150,"tag":169,"props":25633,"children":25635},{"className":25634},[177],[25636,25662],{"type":150,"tag":169,"props":25637,"children":25639},{"className":25638},[182],[25640],{"type":150,"tag":185,"props":25641,"children":25642},{"xmlns":187},[25643],{"type":150,"tag":191,"props":25644,"children":25645},{},[25646,25657],{"type":150,"tag":195,"props":25647,"children":25648},{},[25649,25653],{"type":150,"tag":411,"props":25650,"children":25651},{},[25652],{"type":155,"value":433},{"type":150,"tag":405,"props":25654,"children":25655},{},[25656],{"type":155,"value":25172},{"type":150,"tag":223,"props":25658,"children":25659},{"encoding":225},[25660],{"type":155,"value":25661},"2P",{"type":150,"tag":169,"props":25663,"children":25665},{"className":25664,"ariaHidden":214},[232],[25666],{"type":150,"tag":169,"props":25667,"children":25669},{"className":25668},[237],[25670,25674,25679],{"type":150,"tag":169,"props":25671,"children":25673},{"className":25672,"style":5854},[242],[],{"type":150,"tag":169,"props":25675,"children":25677},{"className":25676},[248],[25678],{"type":155,"value":433},{"type":150,"tag":169,"props":25680,"children":25682},{"className":25681,"style":3869},[248,489],[25683],{"type":155,"value":25172},{"type":155,"value":25685},"，加上 attention 那部分。分母是要搬的字节：权重 ",{"type":150,"tag":169,"props":25687,"children":25689},{"className":25688},[177],[25690,25711],{"type":150,"tag":169,"props":25691,"children":25693},{"className":25692},[182],[25694],{"type":150,"tag":185,"props":25695,"children":25696},{"xmlns":187},[25697],{"type":150,"tag":191,"props":25698,"children":25699},{},[25700,25707],{"type":150,"tag":195,"props":25701,"children":25702},{},[25703],{"type":150,"tag":405,"props":25704,"children":25705},{},[25706],{"type":155,"value":3443},{"type":150,"tag":223,"props":25708,"children":25709},{"encoding":225},[25710],{"type":155,"value":3443},{"type":150,"tag":169,"props":25712,"children":25714},{"className":25713,"ariaHidden":214},[232],[25715],{"type":150,"tag":169,"props":25716,"children":25718},{"className":25717},[237],[25719,25723],{"type":150,"tag":169,"props":25720,"children":25722},{"className":25721,"style":5854},[242],[],{"type":150,"tag":169,"props":25724,"children":25726},{"className":25725,"style":3869},[248,489],[25727],{"type":155,"value":3443},{"type":155,"value":25729},"，加上 KV。代进数字：",{"type":150,"tag":169,"props":25731,"children":25733},{"className":25732},[172],[25734],{"type":150,"tag":169,"props":25735,"children":25737},{"className":25736},[177],[25738,25873],{"type":150,"tag":169,"props":25739,"children":25741},{"className":25740},[182],[25742],{"type":150,"tag":185,"props":25743,"children":25744},{"xmlns":187,"display":188},[25745],{"type":150,"tag":191,"props":25746,"children":25747},{},[25748,25868],{"type":150,"tag":195,"props":25749,"children":25750},{},[25751,25755,25759,25763,25767,25771],{"type":150,"tag":217,"props":25752,"children":25753},{},[25754],{"type":155,"value":25141},{"type":150,"tag":417,"props":25756,"children":25757},{"stretchy":207},[25758],{"type":155,"value":972},{"type":150,"tag":405,"props":25760,"children":25761},{},[25762],{"type":155,"value":5591},{"type":150,"tag":417,"props":25764,"children":25765},{"stretchy":207},[25766],{"type":155,"value":982},{"type":150,"tag":417,"props":25768,"children":25769},{},[25770],{"type":155,"value":1045},{"type":150,"tag":1952,"props":25772,"children":25773},{},[25774,25822],{"type":150,"tag":195,"props":25775,"children":25776},{},[25777,25781,25785,25798,25802,25806,25810,25814,25818],{"type":150,"tag":411,"props":25778,"children":25779},{},[25780],{"type":155,"value":24940},{"type":150,"tag":417,"props":25782,"children":25783},{},[25784],{"type":155,"value":5554},{"type":150,"tag":746,"props":25786,"children":25787},{},[25788,25793],{"type":150,"tag":411,"props":25789,"children":25790},{},[25791],{"type":155,"value":25792},"10",{"type":150,"tag":411,"props":25794,"children":25795},{},[25796],{"type":155,"value":25797},"9",{"type":150,"tag":417,"props":25799,"children":25800},{},[25801],{"type":155,"value":5136},{"type":150,"tag":411,"props":25803,"children":25804},{},[25805],{"type":155,"value":23535},{"type":150,"tag":417,"props":25807,"children":25808},{},[25809],{"type":155,"value":5554},{"type":150,"tag":411,"props":25811,"children":25812},{},[25813],{"type":155,"value":21701},{"type":150,"tag":417,"props":25815,"children":25816},{},[25817],{"type":155,"value":5554},{"type":150,"tag":405,"props":25819,"children":25820},{},[25821],{"type":155,"value":5591},{"type":150,"tag":195,"props":25823,"children":25824},{},[25825,25829,25833,25844,25848,25852,25856,25860,25864],{"type":150,"tag":411,"props":25826,"children":25827},{},[25828],{"type":155,"value":24940},{"type":150,"tag":417,"props":25830,"children":25831},{},[25832],{"type":155,"value":5554},{"type":150,"tag":746,"props":25834,"children":25835},{},[25836,25840],{"type":150,"tag":411,"props":25837,"children":25838},{},[25839],{"type":155,"value":25792},{"type":150,"tag":411,"props":25841,"children":25842},{},[25843],{"type":155,"value":25797},{"type":150,"tag":417,"props":25845,"children":25846},{},[25847],{"type":155,"value":5136},{"type":150,"tag":411,"props":25849,"children":25850},{},[25851],{"type":155,"value":23408},{"type":150,"tag":417,"props":25853,"children":25854},{},[25855],{"type":155,"value":5554},{"type":150,"tag":411,"props":25857,"children":25858},{},[25859],{"type":155,"value":21701},{"type":150,"tag":417,"props":25861,"children":25862},{},[25863],{"type":155,"value":5554},{"type":150,"tag":405,"props":25865,"children":25866},{},[25867],{"type":155,"value":5591},{"type":150,"tag":223,"props":25869,"children":25870},{"encoding":225},[25871],{"type":155,"value":25872},"\\text{AI}(N) = \\frac{42 \\times 10^9 + 278528 \\times 60 \\times N}{42 \\times 10^9 + 1152 \\times 60 \\times N}",{"type":150,"tag":169,"props":25874,"children":25876},{"className":25875,"ariaHidden":214},[232],[25877,25922],{"type":150,"tag":169,"props":25878,"children":25880},{"className":25879},[237],[25881,25885,25894,25899,25904,25909,25913,25918],{"type":150,"tag":169,"props":25882,"children":25884},{"className":25883,"style":999},[242],[],{"type":150,"tag":169,"props":25886,"children":25888},{"className":25887},[248,155],[25889],{"type":150,"tag":169,"props":25890,"children":25892},{"className":25891},[248],[25893],{"type":155,"value":25141},{"type":150,"tag":169,"props":25895,"children":25897},{"className":25896},[1004],[25898],{"type":155,"value":972},{"type":150,"tag":169,"props":25900,"children":25902},{"className":25901,"style":5780},[248,489],[25903],{"type":155,"value":5591},{"type":150,"tag":169,"props":25905,"children":25907},{"className":25906},[1015],[25908],{"type":155,"value":982},{"type":150,"tag":169,"props":25910,"children":25912},{"className":25911,"style":839},[558],[],{"type":150,"tag":169,"props":25914,"children":25916},{"className":25915},[844],[25917],{"type":155,"value":1045},{"type":150,"tag":169,"props":25919,"children":25921},{"className":25920,"style":839},[558],[],{"type":150,"tag":169,"props":25923,"children":25925},{"className":25924},[237],[25926,25931],{"type":150,"tag":169,"props":25927,"children":25930},{"className":25928,"style":25929},[242],"height:2.2604em;vertical-align:-0.7693em;",[],{"type":150,"tag":169,"props":25932,"children":25934},{"className":25933},[248],[25935,25939,26244],{"type":150,"tag":169,"props":25936,"children":25938},{"className":25937},[1004,2660],[],{"type":150,"tag":169,"props":25940,"children":25942},{"className":25941},[1952],[25943],{"type":150,"tag":169,"props":25944,"children":25946},{"className":25945},[253,254],[25947,26232],{"type":150,"tag":169,"props":25948,"children":25950},{"className":25949},[259],[25951,26227],{"type":150,"tag":169,"props":25952,"children":25955},{"className":25953,"style":25954},[264],"height:1.4911em;",[25956,26087,26098],{"type":150,"tag":169,"props":25957,"children":25958},{"style":8422},[25959,25963],{"type":150,"tag":169,"props":25960,"children":25962},{"className":25961,"style":8427},[274],[],{"type":150,"tag":169,"props":25964,"children":25966},{"className":25965},[248],[25967,25972,25976,25981,25985,25990,26033,26037,26042,26046,26051,26055,26060,26064,26069,26073,26078,26082],{"type":150,"tag":169,"props":25968,"children":25970},{"className":25969},[248],[25971],{"type":155,"value":24940},{"type":150,"tag":169,"props":25973,"children":25975},{"className":25974,"style":5167},[558],[],{"type":150,"tag":169,"props":25977,"children":25979},{"className":25978},[5172],[25980],{"type":155,"value":5554},{"type":150,"tag":169,"props":25982,"children":25984},{"className":25983,"style":5167},[558],[],{"type":150,"tag":169,"props":25986,"children":25988},{"className":25987},[248],[25989],{"type":155,"value":415},{"type":150,"tag":169,"props":25991,"children":25993},{"className":25992},[248],[25994,25999],{"type":150,"tag":169,"props":25995,"children":25997},{"className":25996},[248],[25998],{"type":155,"value":206},{"type":150,"tag":169,"props":26000,"children":26002},{"className":26001},[495],[26003],{"type":150,"tag":169,"props":26004,"children":26006},{"className":26005},[253],[26007],{"type":150,"tag":169,"props":26008,"children":26010},{"className":26009},[259],[26011],{"type":150,"tag":169,"props":26012,"children":26015},{"className":26013,"style":26014},[264],"height:0.7401em;",[26016],{"type":150,"tag":169,"props":26017,"children":26019},{"style":26018},"top:-2.989em;margin-right:0.05em;",[26020,26024],{"type":150,"tag":169,"props":26021,"children":26023},{"className":26022,"style":517},[274],[],{"type":150,"tag":169,"props":26025,"children":26027},{"className":26026},[522,523,524,525],[26028],{"type":150,"tag":169,"props":26029,"children":26031},{"className":26030},[248,525],[26032],{"type":155,"value":25797},{"type":150,"tag":169,"props":26034,"children":26036},{"className":26035,"style":5167},[558],[],{"type":150,"tag":169,"props":26038,"children":26040},{"className":26039},[5172],[26041],{"type":155,"value":5136},{"type":150,"tag":169,"props":26043,"children":26045},{"className":26044,"style":5167},[558],[],{"type":150,"tag":169,"props":26047,"children":26049},{"className":26048},[248],[26050],{"type":155,"value":23408},{"type":150,"tag":169,"props":26052,"children":26054},{"className":26053,"style":5167},[558],[],{"type":150,"tag":169,"props":26056,"children":26058},{"className":26057},[5172],[26059],{"type":155,"value":5554},{"type":150,"tag":169,"props":26061,"children":26063},{"className":26062,"style":5167},[558],[],{"type":150,"tag":169,"props":26065,"children":26067},{"className":26066},[248],[26068],{"type":155,"value":21701},{"type":150,"tag":169,"props":26070,"children":26072},{"className":26071,"style":5167},[558],[],{"type":150,"tag":169,"props":26074,"children":26076},{"className":26075},[5172],[26077],{"type":155,"value":5554},{"type":150,"tag":169,"props":26079,"children":26081},{"className":26080,"style":5167},[558],[],{"type":150,"tag":169,"props":26083,"children":26085},{"className":26084,"style":5780},[248,489],[26086],{"type":155,"value":5591},{"type":150,"tag":169,"props":26088,"children":26089},{"style":8440},[26090,26094],{"type":150,"tag":169,"props":26091,"children":26093},{"className":26092,"style":8427},[274],[],{"type":150,"tag":169,"props":26095,"children":26097},{"className":26096,"style":2982},[2981],[],{"type":150,"tag":169,"props":26099,"children":26100},{"style":8452},[26101,26105],{"type":150,"tag":169,"props":26102,"children":26104},{"className":26103,"style":8427},[274],[],{"type":150,"tag":169,"props":26106,"children":26108},{"className":26107},[248],[26109,26114,26118,26123,26127,26132,26173,26177,26182,26186,26191,26195,26200,26204,26209,26213,26218,26222],{"type":150,"tag":169,"props":26110,"children":26112},{"className":26111},[248],[26113],{"type":155,"value":24940},{"type":150,"tag":169,"props":26115,"children":26117},{"className":26116,"style":5167},[558],[],{"type":150,"tag":169,"props":26119,"children":26121},{"className":26120},[5172],[26122],{"type":155,"value":5554},{"type":150,"tag":169,"props":26124,"children":26126},{"className":26125,"style":5167},[558],[],{"type":150,"tag":169,"props":26128,"children":26130},{"className":26129},[248],[26131],{"type":155,"value":415},{"type":150,"tag":169,"props":26133,"children":26135},{"className":26134},[248],[26136,26141],{"type":150,"tag":169,"props":26137,"children":26139},{"className":26138},[248],[26140],{"type":155,"value":206},{"type":150,"tag":169,"props":26142,"children":26144},{"className":26143},[495],[26145],{"type":150,"tag":169,"props":26146,"children":26148},{"className":26147},[253],[26149],{"type":150,"tag":169,"props":26150,"children":26152},{"className":26151},[259],[26153],{"type":150,"tag":169,"props":26154,"children":26156},{"className":26155,"style":7005},[264],[26157],{"type":150,"tag":169,"props":26158,"children":26159},{"style":888},[26160,26164],{"type":150,"tag":169,"props":26161,"children":26163},{"className":26162,"style":517},[274],[],{"type":150,"tag":169,"props":26165,"children":26167},{"className":26166},[522,523,524,525],[26168],{"type":150,"tag":169,"props":26169,"children":26171},{"className":26170},[248,525],[26172],{"type":155,"value":25797},{"type":150,"tag":169,"props":26174,"children":26176},{"className":26175,"style":5167},[558],[],{"type":150,"tag":169,"props":26178,"children":26180},{"className":26179},[5172],[26181],{"type":155,"value":5136},{"type":150,"tag":169,"props":26183,"children":26185},{"className":26184,"style":5167},[558],[],{"type":150,"tag":169,"props":26187,"children":26189},{"className":26188},[248],[26190],{"type":155,"value":23535},{"type":150,"tag":169,"props":26192,"children":26194},{"className":26193,"style":5167},[558],[],{"type":150,"tag":169,"props":26196,"children":26198},{"className":26197},[5172],[26199],{"type":155,"value":5554},{"type":150,"tag":169,"props":26201,"children":26203},{"className":26202,"style":5167},[558],[],{"type":150,"tag":169,"props":26205,"children":26207},{"className":26206},[248],[26208],{"type":155,"value":21701},{"type":150,"tag":169,"props":26210,"children":26212},{"className":26211,"style":5167},[558],[],{"type":150,"tag":169,"props":26214,"children":26216},{"className":26215},[5172],[26217],{"type":155,"value":5554},{"type":150,"tag":169,"props":26219,"children":26221},{"className":26220,"style":5167},[558],[],{"type":150,"tag":169,"props":26223,"children":26225},{"className":26224,"style":5780},[248,489],[26226],{"type":155,"value":5591},{"type":150,"tag":169,"props":26228,"children":26230},{"className":26229},[355],[26231],{"type":155,"value":358},{"type":150,"tag":169,"props":26233,"children":26235},{"className":26234},[259],[26236],{"type":150,"tag":169,"props":26237,"children":26240},{"className":26238,"style":26239},[264],"height:0.7693em;",[26241],{"type":150,"tag":169,"props":26242,"children":26243},{},[],{"type":150,"tag":169,"props":26245,"children":26247},{"className":26246},[1015,2660],[],{"type":150,"tag":151,"props":26249,"children":26250},{},[26251,26293],{"type":150,"tag":169,"props":26252,"children":26254},{"className":26253},[177],[26255,26276],{"type":150,"tag":169,"props":26256,"children":26258},{"className":26257},[182],[26259],{"type":150,"tag":185,"props":26260,"children":26261},{"xmlns":187},[26262],{"type":150,"tag":191,"props":26263,"children":26264},{},[26265,26272],{"type":150,"tag":195,"props":26266,"children":26267},{},[26268],{"type":150,"tag":405,"props":26269,"children":26270},{},[26271],{"type":155,"value":5591},{"type":150,"tag":223,"props":26273,"children":26274},{"encoding":225},[26275],{"type":155,"value":5591},{"type":150,"tag":169,"props":26277,"children":26279},{"className":26278,"ariaHidden":214},[232],[26280],{"type":150,"tag":169,"props":26281,"children":26283},{"className":26282},[237],[26284,26288],{"type":150,"tag":169,"props":26285,"children":26287},{"className":26286,"style":5854},[242],[],{"type":150,"tag":169,"props":26289,"children":26291},{"className":26290,"style":5780},[248,489],[26292],{"type":155,"value":5591},{"type":155,"value":26294}," 一大，分子分母里那两项权重就被 KV 和 attention 项压过去了，于是有上界：",{"type":150,"tag":169,"props":26296,"children":26298},{"className":26297},[172],[26299],{"type":150,"tag":169,"props":26300,"children":26302},{"className":26301},[177],[26303,26425],{"type":150,"tag":169,"props":26304,"children":26306},{"className":26305},[182],[26307],{"type":150,"tag":185,"props":26308,"children":26309},{"xmlns":187,"display":188},[26310],{"type":150,"tag":191,"props":26311,"children":26312},{},[26313,26420],{"type":150,"tag":195,"props":26314,"children":26315},{},[26316,26347,26351,26355,26359,26363,26367,26392,26396,26407,26411,26416],{"type":150,"tag":1959,"props":26317,"children":26318},{},[26319,26331],{"type":150,"tag":195,"props":26320,"children":26321},{},[26322,26327],{"type":150,"tag":405,"props":26323,"children":26324},{},[26325],{"type":155,"value":26326},"lim",{"type":150,"tag":417,"props":26328,"children":26329},{},[26330],{"type":155,"value":1991},{"type":150,"tag":195,"props":26332,"children":26333},{},[26334,26338,26342],{"type":150,"tag":405,"props":26335,"children":26336},{},[26337],{"type":155,"value":5591},{"type":150,"tag":417,"props":26339,"children":26340},{},[26341],{"type":155,"value":12954},{"type":150,"tag":405,"props":26343,"children":26344},{"mathvariant":1796},[26345],{"type":155,"value":26346},"∞",{"type":150,"tag":217,"props":26348,"children":26349},{},[26350],{"type":155,"value":25141},{"type":150,"tag":417,"props":26352,"children":26353},{"stretchy":207},[26354],{"type":155,"value":972},{"type":150,"tag":405,"props":26356,"children":26357},{},[26358],{"type":155,"value":5591},{"type":150,"tag":417,"props":26360,"children":26361},{"stretchy":207},[26362],{"type":155,"value":982},{"type":150,"tag":417,"props":26364,"children":26365},{},[26366],{"type":155,"value":1045},{"type":150,"tag":1952,"props":26368,"children":26369},{},[26370,26381],{"type":150,"tag":401,"props":26371,"children":26372},{},[26373,26377],{"type":150,"tag":405,"props":26374,"children":26375},{},[26376],{"type":155,"value":10046},{"type":150,"tag":217,"props":26378,"children":26379},{},[26380],{"type":155,"value":24428},{"type":150,"tag":401,"props":26382,"children":26383},{},[26384,26388],{"type":150,"tag":405,"props":26385,"children":26386},{},[26387],{"type":155,"value":10046},{"type":150,"tag":217,"props":26389,"children":26390},{},[26391],{"type":155,"value":23996},{"type":150,"tag":417,"props":26393,"children":26394},{},[26395],{"type":155,"value":1045},{"type":150,"tag":1952,"props":26397,"children":26398},{},[26399,26403],{"type":150,"tag":411,"props":26400,"children":26401},{},[26402],{"type":155,"value":23535},{"type":150,"tag":411,"props":26404,"children":26405},{},[26406],{"type":155,"value":23408},{"type":150,"tag":417,"props":26408,"children":26409},{},[26410],{"type":155,"value":6050},{"type":150,"tag":411,"props":26412,"children":26413},{},[26414],{"type":155,"value":26415},"242",{"type":150,"tag":217,"props":26417,"children":26418},{},[26419],{"type":155,"value":23643},{"type":150,"tag":223,"props":26421,"children":26422},{"encoding":225},[26423],{"type":155,"value":26424},"\\lim_{N \\to \\infty} \\text{AI}(N) = \\frac{c_{\\text{flop}}}{c_{\\text{kv}}} = \\frac{278528}{1152} \\approx 242\\ \\text{FLOP\u002Fbyte}",{"type":150,"tag":169,"props":26426,"children":26428},{"className":26427,"ariaHidden":214},[232],[26429,26560,26789,26899],{"type":150,"tag":169,"props":26430,"children":26432},{"className":26431},[237],[26433,26438,26519,26523,26532,26537,26542,26547,26551,26556],{"type":150,"tag":169,"props":26434,"children":26437},{"className":26435,"style":26436},[242],"height:1.4943em;vertical-align:-0.7443em;",[],{"type":150,"tag":169,"props":26439,"children":26442},{"className":26440},[2695,26441],"op-limits",[26443],{"type":150,"tag":169,"props":26444,"children":26446},{"className":26445},[253,254],[26447,26507],{"type":150,"tag":169,"props":26448,"children":26450},{"className":26449},[259],[26451,26502],{"type":150,"tag":169,"props":26452,"children":26454},{"className":26453,"style":942},[264],[26455,26486],{"type":150,"tag":169,"props":26456,"children":26458},{"style":26457},"top:-2.3557em;margin-left:0em;",[26459,26463],{"type":150,"tag":169,"props":26460,"children":26462},{"className":26461,"style":8427},[274],[],{"type":150,"tag":169,"props":26464,"children":26466},{"className":26465},[522,523,524,525],[26467],{"type":150,"tag":169,"props":26468,"children":26470},{"className":26469},[248,525],[26471,26476,26481],{"type":150,"tag":169,"props":26472,"children":26474},{"className":26473,"style":5780},[248,489,525],[26475],{"type":155,"value":5591},{"type":150,"tag":169,"props":26477,"children":26479},{"className":26478},[844,525],[26480],{"type":155,"value":12954},{"type":150,"tag":169,"props":26482,"children":26484},{"className":26483},[248,525],[26485],{"type":155,"value":26346},{"type":150,"tag":169,"props":26487,"children":26489},{"style":26488},"top:-3em;",[26490,26494],{"type":150,"tag":169,"props":26491,"children":26493},{"className":26492,"style":8427},[274],[],{"type":150,"tag":169,"props":26495,"children":26496},{},[26497],{"type":150,"tag":169,"props":26498,"children":26500},{"className":26499},[2695],[26501],{"type":155,"value":26326},{"type":150,"tag":169,"props":26503,"children":26505},{"className":26504},[355],[26506],{"type":155,"value":358},{"type":150,"tag":169,"props":26508,"children":26510},{"className":26509},[259],[26511],{"type":150,"tag":169,"props":26512,"children":26515},{"className":26513,"style":26514},[264],"height:0.7443em;",[26516],{"type":150,"tag":169,"props":26517,"children":26518},{},[],{"type":150,"tag":169,"props":26520,"children":26522},{"className":26521,"style":559},[558],[],{"type":150,"tag":169,"props":26524,"children":26526},{"className":26525},[248,155],[26527],{"type":150,"tag":169,"props":26528,"children":26530},{"className":26529},[248],[26531],{"type":155,"value":25141},{"type":150,"tag":169,"props":26533,"children":26535},{"className":26534},[1004],[26536],{"type":155,"value":972},{"type":150,"tag":169,"props":26538,"children":26540},{"className":26539,"style":5780},[248,489],[26541],{"type":155,"value":5591},{"type":150,"tag":169,"props":26543,"children":26545},{"className":26544},[1015],[26546],{"type":155,"value":982},{"type":150,"tag":169,"props":26548,"children":26550},{"className":26549,"style":839},[558],[],{"type":150,"tag":169,"props":26552,"children":26554},{"className":26553},[844],[26555],{"type":155,"value":1045},{"type":150,"tag":169,"props":26557,"children":26559},{"className":26558,"style":839},[558],[],{"type":150,"tag":169,"props":26561,"children":26563},{"className":26562},[237],[26564,26569,26776,26780,26785],{"type":150,"tag":169,"props":26565,"children":26568},{"className":26566,"style":26567},[242],"height:1.9436em;vertical-align:-0.836em;",[],{"type":150,"tag":169,"props":26570,"children":26572},{"className":26571},[248],[26573,26577,26772],{"type":150,"tag":169,"props":26574,"children":26576},{"className":26575},[1004,2660],[],{"type":150,"tag":169,"props":26578,"children":26580},{"className":26579},[1952],[26581],{"type":150,"tag":169,"props":26582,"children":26584},{"className":26583},[253,254],[26585,26761],{"type":150,"tag":169,"props":26586,"children":26588},{"className":26587},[259],[26589,26756],{"type":150,"tag":169,"props":26590,"children":26592},{"className":26591,"style":8418},[264],[26593,26669,26680],{"type":150,"tag":169,"props":26594,"children":26595},{"style":8422},[26596,26600],{"type":150,"tag":169,"props":26597,"children":26599},{"className":26598,"style":8427},[274],[],{"type":150,"tag":169,"props":26601,"children":26603},{"className":26602},[248],[26604],{"type":150,"tag":169,"props":26605,"children":26607},{"className":26606},[248],[26608,26613],{"type":150,"tag":169,"props":26609,"children":26611},{"className":26610},[248,489],[26612],{"type":155,"value":10046},{"type":150,"tag":169,"props":26614,"children":26616},{"className":26615},[495],[26617],{"type":150,"tag":169,"props":26618,"children":26620},{"className":26619},[253,254],[26621,26658],{"type":150,"tag":169,"props":26622,"children":26624},{"className":26623},[259],[26625,26653],{"type":150,"tag":169,"props":26626,"children":26628},{"className":26627,"style":674},[264],[26629],{"type":150,"tag":169,"props":26630,"children":26631},{"style":512},[26632,26636],{"type":150,"tag":169,"props":26633,"children":26635},{"className":26634,"style":517},[274],[],{"type":150,"tag":169,"props":26637,"children":26639},{"className":26638},[522,523,524,525],[26640],{"type":150,"tag":169,"props":26641,"children":26643},{"className":26642},[248,525],[26644],{"type":150,"tag":169,"props":26645,"children":26647},{"className":26646},[248,155,525],[26648],{"type":150,"tag":169,"props":26649,"children":26651},{"className":26650},[248,525],[26652],{"type":155,"value":23996},{"type":150,"tag":169,"props":26654,"children":26656},{"className":26655},[355],[26657],{"type":155,"value":358},{"type":150,"tag":169,"props":26659,"children":26661},{"className":26660},[259],[26662],{"type":150,"tag":169,"props":26663,"children":26665},{"className":26664,"style":544},[264],[26666],{"type":150,"tag":169,"props":26667,"children":26668},{},[],{"type":150,"tag":169,"props":26670,"children":26671},{"style":8440},[26672,26676],{"type":150,"tag":169,"props":26673,"children":26675},{"className":26674,"style":8427},[274],[],{"type":150,"tag":169,"props":26677,"children":26679},{"className":26678,"style":2982},[2981],[],{"type":150,"tag":169,"props":26681,"children":26682},{"style":8452},[26683,26687],{"type":150,"tag":169,"props":26684,"children":26686},{"className":26685,"style":8427},[274],[],{"type":150,"tag":169,"props":26688,"children":26690},{"className":26689},[248],[26691],{"type":150,"tag":169,"props":26692,"children":26694},{"className":26693},[248],[26695,26700],{"type":150,"tag":169,"props":26696,"children":26698},{"className":26697},[248,489],[26699],{"type":155,"value":10046},{"type":150,"tag":169,"props":26701,"children":26703},{"className":26702},[495],[26704],{"type":150,"tag":169,"props":26705,"children":26707},{"className":26706},[253,254],[26708,26745],{"type":150,"tag":169,"props":26709,"children":26711},{"className":26710},[259],[26712,26740],{"type":150,"tag":169,"props":26713,"children":26715},{"className":26714,"style":674},[264],[26716],{"type":150,"tag":169,"props":26717,"children":26718},{"style":512},[26719,26723],{"type":150,"tag":169,"props":26720,"children":26722},{"className":26721,"style":517},[274],[],{"type":150,"tag":169,"props":26724,"children":26726},{"className":26725},[522,523,524,525],[26727],{"type":150,"tag":169,"props":26728,"children":26730},{"className":26729},[248,525],[26731],{"type":150,"tag":169,"props":26732,"children":26734},{"className":26733},[248,155,525],[26735],{"type":150,"tag":169,"props":26736,"children":26738},{"className":26737},[248,525],[26739],{"type":155,"value":24428},{"type":150,"tag":169,"props":26741,"children":26743},{"className":26742},[355],[26744],{"type":155,"value":358},{"type":150,"tag":169,"props":26746,"children":26748},{"className":26747},[259],[26749],{"type":150,"tag":169,"props":26750,"children":26752},{"className":26751,"style":6692},[264],[26753],{"type":150,"tag":169,"props":26754,"children":26755},{},[],{"type":150,"tag":169,"props":26757,"children":26759},{"className":26758},[355],[26760],{"type":155,"value":358},{"type":150,"tag":169,"props":26762,"children":26764},{"className":26763},[259],[26765],{"type":150,"tag":169,"props":26766,"children":26768},{"className":26767,"style":25618},[264],[26769],{"type":150,"tag":169,"props":26770,"children":26771},{},[],{"type":150,"tag":169,"props":26773,"children":26775},{"className":26774},[1015,2660],[],{"type":150,"tag":169,"props":26777,"children":26779},{"className":26778,"style":839},[558],[],{"type":150,"tag":169,"props":26781,"children":26783},{"className":26782},[844],[26784],{"type":155,"value":1045},{"type":150,"tag":169,"props":26786,"children":26788},{"className":26787,"style":839},[558],[],{"type":150,"tag":169,"props":26790,"children":26792},{"className":26791},[237],[26793,26798,26886,26890,26895],{"type":150,"tag":169,"props":26794,"children":26797},{"className":26795,"style":26796},[242],"height:2.0074em;vertical-align:-0.686em;",[],{"type":150,"tag":169,"props":26799,"children":26801},{"className":26800},[248],[26802,26806,26882],{"type":150,"tag":169,"props":26803,"children":26805},{"className":26804},[1004,2660],[],{"type":150,"tag":169,"props":26807,"children":26809},{"className":26808},[1952],[26810],{"type":150,"tag":169,"props":26811,"children":26813},{"className":26812},[253,254],[26814,26871],{"type":150,"tag":169,"props":26815,"children":26817},{"className":26816},[259],[26818,26866],{"type":150,"tag":169,"props":26819,"children":26822},{"className":26820,"style":26821},[264],"height:1.3214em;",[26823,26839,26850],{"type":150,"tag":169,"props":26824,"children":26825},{"style":8422},[26826,26830],{"type":150,"tag":169,"props":26827,"children":26829},{"className":26828,"style":8427},[274],[],{"type":150,"tag":169,"props":26831,"children":26833},{"className":26832},[248],[26834],{"type":150,"tag":169,"props":26835,"children":26837},{"className":26836},[248],[26838],{"type":155,"value":23408},{"type":150,"tag":169,"props":26840,"children":26841},{"style":8440},[26842,26846],{"type":150,"tag":169,"props":26843,"children":26845},{"className":26844,"style":8427},[274],[],{"type":150,"tag":169,"props":26847,"children":26849},{"className":26848,"style":2982},[2981],[],{"type":150,"tag":169,"props":26851,"children":26852},{"style":8452},[26853,26857],{"type":150,"tag":169,"props":26854,"children":26856},{"className":26855,"style":8427},[274],[],{"type":150,"tag":169,"props":26858,"children":26860},{"className":26859},[248],[26861],{"type":150,"tag":169,"props":26862,"children":26864},{"className":26863},[248],[26865],{"type":155,"value":23535},{"type":150,"tag":169,"props":26867,"children":26869},{"className":26868},[355],[26870],{"type":155,"value":358},{"type":150,"tag":169,"props":26872,"children":26874},{"className":26873},[259],[26875],{"type":150,"tag":169,"props":26876,"children":26878},{"className":26877,"style":8484},[264],[26879],{"type":150,"tag":169,"props":26880,"children":26881},{},[],{"type":150,"tag":169,"props":26883,"children":26885},{"className":26884},[1015,2660],[],{"type":150,"tag":169,"props":26887,"children":26889},{"className":26888,"style":839},[558],[],{"type":150,"tag":169,"props":26891,"children":26893},{"className":26892},[844],[26894],{"type":155,"value":6050},{"type":150,"tag":169,"props":26896,"children":26898},{"className":26897,"style":839},[558],[],{"type":150,"tag":169,"props":26900,"children":26902},{"className":26901},[237],[26903,26907,26912,26917],{"type":150,"tag":169,"props":26904,"children":26906},{"className":26905,"style":999},[242],[],{"type":150,"tag":169,"props":26908,"children":26910},{"className":26909},[248],[26911],{"type":155,"value":26415},{"type":150,"tag":169,"props":26913,"children":26915},{"className":26914},[558],[26916],{"type":155,"value":8763},{"type":150,"tag":169,"props":26918,"children":26920},{"className":26919},[248,155],[26921],{"type":150,"tag":169,"props":26922,"children":26924},{"className":26923},[248],[26925],{"type":155,"value":23821},{"type":150,"tag":151,"props":26927,"children":26928},{},[26929,26931,26973,26975,26980],{"type":155,"value":26930},"注意 ",{"type":150,"tag":169,"props":26932,"children":26934},{"className":26933},[177],[26935,26956],{"type":150,"tag":169,"props":26936,"children":26938},{"className":26937},[182],[26939],{"type":150,"tag":185,"props":26940,"children":26941},{"xmlns":187},[26942],{"type":150,"tag":191,"props":26943,"children":26944},{},[26945,26952],{"type":150,"tag":195,"props":26946,"children":26947},{},[26948],{"type":150,"tag":405,"props":26949,"children":26950},{},[26951],{"type":155,"value":5559},{"type":150,"tag":223,"props":26953,"children":26954},{"encoding":225},[26955],{"type":155,"value":5559},{"type":150,"tag":169,"props":26957,"children":26959},{"className":26958,"ariaHidden":214},[232],[26960],{"type":150,"tag":169,"props":26961,"children":26963},{"className":26962},[237],[26964,26968],{"type":150,"tag":169,"props":26965,"children":26967},{"className":26966,"style":5854},[242],[],{"type":150,"tag":169,"props":26969,"children":26971},{"className":26970},[248,489],[26972],{"type":155,"value":5559},{"type":155,"value":26974}," 被约掉了。",{"type":150,"tag":158,"props":26976,"children":26977},{},[26978],{"type":155,"value":26979},"无论多少层，这个上限都一样",{"type":155,"value":26981},"，因为 KV 和 attention 计算都随层数线性增长。",{"type":150,"tag":151,"props":26983,"children":26984},{},[26985,26990],{"type":150,"tag":158,"props":26986,"children":26987},{},[26988],{"type":155,"value":26989},"242 小于 295。",{"type":155,"value":26991}," 也就是说，不管 context 多长，MLA 的 decode 都落在拐点左边，是 memory bound。那 4.25 倍的计算量加进去就是白加的。算力本来就在空转等内存。",{"type":150,"tag":151,"props":26993,"children":26994},{},[26995],{"type":155,"value":26996},"具体看几个长度：",{"type":150,"tag":22368,"props":26998,"children":26999},{},[27000,27071],{"type":150,"tag":22372,"props":27001,"children":27002},{},[27003],{"type":150,"tag":22376,"props":27004,"children":27005},{},[27006,27051,27056,27061,27066],{"type":150,"tag":22380,"props":27007,"children":27008},{},[27009],{"type":150,"tag":169,"props":27010,"children":27012},{"className":27011},[177],[27013,27034],{"type":150,"tag":169,"props":27014,"children":27016},{"className":27015},[182],[27017],{"type":150,"tag":185,"props":27018,"children":27019},{"xmlns":187},[27020],{"type":150,"tag":191,"props":27021,"children":27022},{},[27023,27030],{"type":150,"tag":195,"props":27024,"children":27025},{},[27026],{"type":150,"tag":405,"props":27027,"children":27028},{},[27029],{"type":155,"value":5591},{"type":150,"tag":223,"props":27031,"children":27032},{"encoding":225},[27033],{"type":155,"value":5591},{"type":150,"tag":169,"props":27035,"children":27037},{"className":27036,"ariaHidden":214},[232],[27038],{"type":150,"tag":169,"props":27039,"children":27041},{"className":27040},[237],[27042,27046],{"type":150,"tag":169,"props":27043,"children":27045},{"className":27044,"style":5854},[242],[],{"type":150,"tag":169,"props":27047,"children":27049},{"className":27048,"style":5780},[248,489],[27050],{"type":155,"value":5591},{"type":150,"tag":22380,"props":27052,"children":27053},{},[27054],{"type":155,"value":27055},"算术强度",{"type":150,"tag":22380,"props":27057,"children":27058},{},[27059],{"type":155,"value":27060},"内存耗时",{"type":150,"tag":22380,"props":27062,"children":27063},{},[27064],{"type":155,"value":27065},"计算耗时",{"type":150,"tag":22380,"props":27067,"children":27068},{},[27069],{"type":155,"value":27070},"瓶颈",{"type":150,"tag":22394,"props":27072,"children":27073},{},[27074,27102,27129,27156],{"type":150,"tag":22376,"props":27075,"children":27076},{},[27077,27082,27087,27092,27097],{"type":150,"tag":22401,"props":27078,"children":27079},{},[27080],{"type":155,"value":27081},"4k",{"type":150,"tag":22401,"props":27083,"children":27084},{},[27085],{"type":155,"value":27086},"2.6",{"type":150,"tag":22401,"props":27088,"children":27089},{},[27090],{"type":155,"value":27091},"12.6 ms",{"type":150,"tag":22401,"props":27093,"children":27094},{},[27095],{"type":155,"value":27096},"0.11 ms",{"type":150,"tag":22401,"props":27098,"children":27099},{},[27100],{"type":155,"value":27101},"memory",{"type":150,"tag":22376,"props":27103,"children":27104},{},[27105,27110,27115,27120,27125],{"type":150,"tag":22401,"props":27106,"children":27107},{},[27108],{"type":155,"value":27109},"32k",{"type":150,"tag":22401,"props":27111,"children":27112},{},[27113],{"type":155,"value":27114},"13.3",{"type":150,"tag":22401,"props":27116,"children":27117},{},[27118],{"type":155,"value":27119},"13.2 ms",{"type":150,"tag":22401,"props":27121,"children":27122},{},[27123],{"type":155,"value":27124},"0.60 ms",{"type":150,"tag":22401,"props":27126,"children":27127},{},[27128],{"type":155,"value":27101},{"type":150,"tag":22376,"props":27130,"children":27131},{},[27132,27137,27142,27147,27152],{"type":150,"tag":22401,"props":27133,"children":27134},{},[27135],{"type":155,"value":27136},"128k",{"type":150,"tag":22401,"props":27138,"children":27139},{},[27140],{"type":155,"value":27141},"43.7",{"type":150,"tag":22401,"props":27143,"children":27144},{},[27145],{"type":155,"value":27146},"15.2 ms",{"type":150,"tag":22401,"props":27148,"children":27149},{},[27150],{"type":155,"value":27151},"2.3 ms",{"type":150,"tag":22401,"props":27153,"children":27154},{},[27155],{"type":155,"value":27101},{"type":150,"tag":22376,"props":27157,"children":27158},{},[27159,27164,27169,27174,27179],{"type":150,"tag":22401,"props":27160,"children":27161},{},[27162],{"type":155,"value":27163},"1M",{"type":150,"tag":22401,"props":27165,"children":27166},{},[27167],{"type":155,"value":27168},"153",{"type":150,"tag":22401,"props":27170,"children":27171},{},[27172],{"type":155,"value":27173},"34.2 ms",{"type":150,"tag":22401,"props":27175,"children":27176},{},[27177],{"type":155,"value":27178},"17.8 ms",{"type":150,"tag":22401,"props":27180,"children":27181},{},[27182],{"type":155,"value":27101},{"type":150,"tag":151,"props":27184,"children":27185},{},[27186],{"type":155,"value":27187},"到 1M 的时候计算耗时（17.8 ms）已经追到内存耗时（34.2 ms）的一半了，但还没越过去。",{"type":150,"tag":151,"props":27189,"children":27190},{},[27191],{"type":155,"value":27192},"不过这里只算了 batch size 等于 1。在线服务几乎不会单条跑，batch 是提高算术强度的另一条路子，而且它会改变结论。",{"type":150,"tag":151,"props":27194,"children":27195},{},[27196,27198,27241],{"type":155,"value":27197},"原因很直接：权重每一步只需要读一次，然后同时喂给 batch 里所有序列；KV 读取却是一条序列一份。把 ",{"type":150,"tag":169,"props":27199,"children":27201},{"className":27200},[177],[27202,27223],{"type":150,"tag":169,"props":27203,"children":27205},{"className":27204},[182],[27206],{"type":150,"tag":185,"props":27207,"children":27208},{"xmlns":187},[27209],{"type":150,"tag":191,"props":27210,"children":27211},{},[27212,27219],{"type":150,"tag":195,"props":27213,"children":27214},{},[27215],{"type":150,"tag":405,"props":27216,"children":27217},{},[27218],{"type":155,"value":24918},{"type":150,"tag":223,"props":27220,"children":27221},{"encoding":225},[27222],{"type":155,"value":24918},{"type":150,"tag":169,"props":27224,"children":27226},{"className":27225,"ariaHidden":214},[232],[27227],{"type":150,"tag":169,"props":27228,"children":27230},{"className":27229},[237],[27231,27235],{"type":150,"tag":169,"props":27232,"children":27234},{"className":27233,"style":5854},[242],[],{"type":150,"tag":169,"props":27236,"children":27239},{"className":27237,"style":27238},[248,489],"margin-right:0.0502em;",[27240],{"type":155,"value":24918},{"type":155,"value":27242}," 写进去：",{"type":150,"tag":169,"props":27244,"children":27246},{"className":27245},[172],[27247],{"type":150,"tag":169,"props":27248,"children":27250},{"className":27249},[177],[27251,27387],{"type":150,"tag":169,"props":27252,"children":27254},{"className":27253},[182],[27255],{"type":150,"tag":185,"props":27256,"children":27257},{"xmlns":187,"display":188},[27258],{"type":150,"tag":191,"props":27259,"children":27260},{},[27261,27382],{"type":150,"tag":195,"props":27262,"children":27263},{},[27264,27268,27272,27276,27280,27284,27288,27292],{"type":150,"tag":217,"props":27265,"children":27266},{},[27267],{"type":155,"value":25141},{"type":150,"tag":417,"props":27269,"children":27270},{"stretchy":207},[27271],{"type":155,"value":972},{"type":150,"tag":405,"props":27273,"children":27274},{},[27275],{"type":155,"value":24918},{"type":150,"tag":417,"props":27277,"children":27278},{"separator":214},[27279],{"type":155,"value":421},{"type":150,"tag":405,"props":27281,"children":27282},{},[27283],{"type":155,"value":5591},{"type":150,"tag":417,"props":27285,"children":27286},{"stretchy":207},[27287],{"type":155,"value":982},{"type":150,"tag":417,"props":27289,"children":27290},{},[27291],{"type":155,"value":1045},{"type":150,"tag":1952,"props":27293,"children":27294},{},[27295,27344],{"type":150,"tag":195,"props":27296,"children":27297},{},[27298,27302],{"type":150,"tag":405,"props":27299,"children":27300},{},[27301],{"type":155,"value":24918},{"type":150,"tag":195,"props":27303,"children":27304},{},[27305,27309,27313,27317,27321,27332,27336,27340],{"type":150,"tag":417,"props":27306,"children":27307},{"fence":214},[27308],{"type":155,"value":972},{"type":150,"tag":411,"props":27310,"children":27311},{},[27312],{"type":155,"value":433},{"type":150,"tag":405,"props":27314,"children":27315},{},[27316],{"type":155,"value":25172},{"type":150,"tag":417,"props":27318,"children":27319},{},[27320],{"type":155,"value":5136},{"type":150,"tag":401,"props":27322,"children":27323},{},[27324,27328],{"type":150,"tag":405,"props":27325,"children":27326},{},[27327],{"type":155,"value":10046},{"type":150,"tag":217,"props":27329,"children":27330},{},[27331],{"type":155,"value":24428},{"type":150,"tag":405,"props":27333,"children":27334},{},[27335],{"type":155,"value":5559},{"type":150,"tag":405,"props":27337,"children":27338},{},[27339],{"type":155,"value":5591},{"type":150,"tag":417,"props":27341,"children":27342},{"fence":214},[27343],{"type":155,"value":982},{"type":150,"tag":195,"props":27345,"children":27346},{},[27347,27351,27355,27359,27363,27374,27378],{"type":150,"tag":405,"props":27348,"children":27349},{},[27350],{"type":155,"value":3443},{"type":150,"tag":417,"props":27352,"children":27353},{},[27354],{"type":155,"value":5136},{"type":150,"tag":405,"props":27356,"children":27357},{},[27358],{"type":155,"value":24918},{"type":150,"tag":417,"props":27360,"children":27361},{},[27362],{"type":155,"value":25192},{"type":150,"tag":401,"props":27364,"children":27365},{},[27366,27370],{"type":150,"tag":405,"props":27367,"children":27368},{},[27369],{"type":155,"value":10046},{"type":150,"tag":217,"props":27371,"children":27372},{},[27373],{"type":155,"value":23996},{"type":150,"tag":405,"props":27375,"children":27376},{},[27377],{"type":155,"value":5559},{"type":150,"tag":405,"props":27379,"children":27380},{},[27381],{"type":155,"value":5591},{"type":150,"tag":223,"props":27383,"children":27384},{"encoding":225},[27385],{"type":155,"value":27386},"\\text{AI}(B, N) = \\frac{B\\left(2P + c_{\\text{flop}} L N\\right)}{W + B \\cdot c_{\\text{kv}} L N}",{"type":150,"tag":169,"props":27388,"children":27390},{"className":27389,"ariaHidden":214},[232],[27391,27450],{"type":150,"tag":169,"props":27392,"children":27394},{"className":27393},[237],[27395,27399,27408,27413,27418,27423,27427,27432,27437,27441,27446],{"type":150,"tag":169,"props":27396,"children":27398},{"className":27397,"style":999},[242],[],{"type":150,"tag":169,"props":27400,"children":27402},{"className":27401},[248,155],[27403],{"type":150,"tag":169,"props":27404,"children":27406},{"className":27405},[248],[27407],{"type":155,"value":25141},{"type":150,"tag":169,"props":27409,"children":27411},{"className":27410},[1004],[27412],{"type":155,"value":972},{"type":150,"tag":169,"props":27414,"children":27416},{"className":27415,"style":27238},[248,489],[27417],{"type":155,"value":24918},{"type":150,"tag":169,"props":27419,"children":27421},{"className":27420},[552],[27422],{"type":155,"value":421},{"type":150,"tag":169,"props":27424,"children":27426},{"className":27425,"style":559},[558],[],{"type":150,"tag":169,"props":27428,"children":27430},{"className":27429,"style":5780},[248,489],[27431],{"type":155,"value":5591},{"type":150,"tag":169,"props":27433,"children":27435},{"className":27434},[1015],[27436],{"type":155,"value":982},{"type":150,"tag":169,"props":27438,"children":27440},{"className":27439,"style":839},[558],[],{"type":150,"tag":169,"props":27442,"children":27444},{"className":27443},[844],[27445],{"type":155,"value":1045},{"type":150,"tag":169,"props":27447,"children":27449},{"className":27448,"style":839},[558],[],{"type":150,"tag":169,"props":27451,"children":27453},{"className":27452},[237],[27454,27459],{"type":150,"tag":169,"props":27455,"children":27458},{"className":27456,"style":27457},[242],"height:2.263em;vertical-align:-0.836em;",[],{"type":150,"tag":169,"props":27460,"children":27462},{"className":27461},[248],[27463,27467,27765],{"type":150,"tag":169,"props":27464,"children":27466},{"className":27465},[1004,2660],[],{"type":150,"tag":169,"props":27468,"children":27470},{"className":27469},[1952],[27471],{"type":150,"tag":169,"props":27472,"children":27474},{"className":27473},[253,254],[27475,27754],{"type":150,"tag":169,"props":27476,"children":27478},{"className":27477},[259],[27479,27749],{"type":150,"tag":169,"props":27480,"children":27483},{"className":27481,"style":27482},[264],"height:1.427em;",[27484,27606,27617],{"type":150,"tag":169,"props":27485,"children":27486},{"style":8422},[27487,27491],{"type":150,"tag":169,"props":27488,"children":27490},{"className":27489,"style":8427},[274],[],{"type":150,"tag":169,"props":27492,"children":27494},{"className":27493},[248],[27495,27500,27504,27509,27513,27518,27522,27527,27531,27596,27601],{"type":150,"tag":169,"props":27496,"children":27498},{"className":27497,"style":3869},[248,489],[27499],{"type":155,"value":3443},{"type":150,"tag":169,"props":27501,"children":27503},{"className":27502,"style":5167},[558],[],{"type":150,"tag":169,"props":27505,"children":27507},{"className":27506},[5172],[27508],{"type":155,"value":5136},{"type":150,"tag":169,"props":27510,"children":27512},{"className":27511,"style":5167},[558],[],{"type":150,"tag":169,"props":27514,"children":27516},{"className":27515,"style":27238},[248,489],[27517],{"type":155,"value":24918},{"type":150,"tag":169,"props":27519,"children":27521},{"className":27520,"style":5167},[558],[],{"type":150,"tag":169,"props":27523,"children":27525},{"className":27524},[5172],[27526],{"type":155,"value":25192},{"type":150,"tag":169,"props":27528,"children":27530},{"className":27529,"style":5167},[558],[],{"type":150,"tag":169,"props":27532,"children":27534},{"className":27533},[248],[27535,27540],{"type":150,"tag":169,"props":27536,"children":27538},{"className":27537},[248,489],[27539],{"type":155,"value":10046},{"type":150,"tag":169,"props":27541,"children":27543},{"className":27542},[495],[27544],{"type":150,"tag":169,"props":27545,"children":27547},{"className":27546},[253,254],[27548,27585],{"type":150,"tag":169,"props":27549,"children":27551},{"className":27550},[259],[27552,27580],{"type":150,"tag":169,"props":27553,"children":27555},{"className":27554,"style":674},[264],[27556],{"type":150,"tag":169,"props":27557,"children":27558},{"style":512},[27559,27563],{"type":150,"tag":169,"props":27560,"children":27562},{"className":27561,"style":517},[274],[],{"type":150,"tag":169,"props":27564,"children":27566},{"className":27565},[522,523,524,525],[27567],{"type":150,"tag":169,"props":27568,"children":27570},{"className":27569},[248,525],[27571],{"type":150,"tag":169,"props":27572,"children":27574},{"className":27573},[248,155,525],[27575],{"type":150,"tag":169,"props":27576,"children":27578},{"className":27577},[248,525],[27579],{"type":155,"value":23996},{"type":150,"tag":169,"props":27581,"children":27583},{"className":27582},[355],[27584],{"type":155,"value":358},{"type":150,"tag":169,"props":27586,"children":27588},{"className":27587},[259],[27589],{"type":150,"tag":169,"props":27590,"children":27592},{"className":27591,"style":544},[264],[27593],{"type":150,"tag":169,"props":27594,"children":27595},{},[],{"type":150,"tag":169,"props":27597,"children":27599},{"className":27598},[248,489],[27600],{"type":155,"value":5559},{"type":150,"tag":169,"props":27602,"children":27604},{"className":27603,"style":5780},[248,489],[27605],{"type":155,"value":5591},{"type":150,"tag":169,"props":27607,"children":27608},{"style":8440},[27609,27613],{"type":150,"tag":169,"props":27610,"children":27612},{"className":27611,"style":8427},[274],[],{"type":150,"tag":169,"props":27614,"children":27616},{"className":27615,"style":2982},[2981],[],{"type":150,"tag":169,"props":27618,"children":27619},{"style":8452},[27620,27624],{"type":150,"tag":169,"props":27621,"children":27623},{"className":27622,"style":8427},[274],[],{"type":150,"tag":169,"props":27625,"children":27627},{"className":27626},[248],[27628,27633,27637],{"type":150,"tag":169,"props":27629,"children":27631},{"className":27630,"style":27238},[248,489],[27632],{"type":155,"value":24918},{"type":150,"tag":169,"props":27634,"children":27636},{"className":27635,"style":559},[558],[],{"type":150,"tag":169,"props":27638,"children":27640},{"className":27639},[630],[27641,27646,27651,27656,27660,27665,27669,27734,27739,27744],{"type":150,"tag":169,"props":27642,"children":27644},{"className":27643,"style":8386},[1004,8385],[27645],{"type":155,"value":972},{"type":150,"tag":169,"props":27647,"children":27649},{"className":27648},[248],[27650],{"type":155,"value":433},{"type":150,"tag":169,"props":27652,"children":27654},{"className":27653,"style":3869},[248,489],[27655],{"type":155,"value":25172},{"type":150,"tag":169,"props":27657,"children":27659},{"className":27658,"style":5167},[558],[],{"type":150,"tag":169,"props":27661,"children":27663},{"className":27662},[5172],[27664],{"type":155,"value":5136},{"type":150,"tag":169,"props":27666,"children":27668},{"className":27667,"style":5167},[558],[],{"type":150,"tag":169,"props":27670,"children":27672},{"className":27671},[248],[27673,27678],{"type":150,"tag":169,"props":27674,"children":27676},{"className":27675},[248,489],[27677],{"type":155,"value":10046},{"type":150,"tag":169,"props":27679,"children":27681},{"className":27680},[495],[27682],{"type":150,"tag":169,"props":27683,"children":27685},{"className":27684},[253,254],[27686,27723],{"type":150,"tag":169,"props":27687,"children":27689},{"className":27688},[259],[27690,27718],{"type":150,"tag":169,"props":27691,"children":27693},{"className":27692,"style":674},[264],[27694],{"type":150,"tag":169,"props":27695,"children":27696},{"style":512},[27697,27701],{"type":150,"tag":169,"props":27698,"children":27700},{"className":27699,"style":517},[274],[],{"type":150,"tag":169,"props":27702,"children":27704},{"className":27703},[522,523,524,525],[27705],{"type":150,"tag":169,"props":27706,"children":27708},{"className":27707},[248,525],[27709],{"type":150,"tag":169,"props":27710,"children":27712},{"className":27711},[248,155,525],[27713],{"type":150,"tag":169,"props":27714,"children":27716},{"className":27715},[248,525],[27717],{"type":155,"value":24428},{"type":150,"tag":169,"props":27719,"children":27721},{"className":27720},[355],[27722],{"type":155,"value":358},{"type":150,"tag":169,"props":27724,"children":27726},{"className":27725},[259],[27727],{"type":150,"tag":169,"props":27728,"children":27730},{"className":27729,"style":6692},[264],[27731],{"type":150,"tag":169,"props":27732,"children":27733},{},[],{"type":150,"tag":169,"props":27735,"children":27737},{"className":27736},[248,489],[27738],{"type":155,"value":5559},{"type":150,"tag":169,"props":27740,"children":27742},{"className":27741,"style":5780},[248,489],[27743],{"type":155,"value":5591},{"type":150,"tag":169,"props":27745,"children":27747},{"className":27746,"style":8386},[1015,8385],[27748],{"type":155,"value":982},{"type":150,"tag":169,"props":27750,"children":27752},{"className":27751},[355],[27753],{"type":155,"value":358},{"type":150,"tag":169,"props":27755,"children":27757},{"className":27756},[259],[27758],{"type":150,"tag":169,"props":27759,"children":27761},{"className":27760,"style":25618},[264],[27762],{"type":150,"tag":169,"props":27763,"children":27764},{},[],{"type":150,"tag":169,"props":27766,"children":27768},{"className":27767},[1015,2660],[],{"type":150,"tag":151,"props":27770,"children":27771},{},[27772,27814,27816,27858],{"type":150,"tag":169,"props":27773,"children":27775},{"className":27774},[177],[27776,27797],{"type":150,"tag":169,"props":27777,"children":27779},{"className":27778},[182],[27780],{"type":150,"tag":185,"props":27781,"children":27782},{"xmlns":187},[27783],{"type":150,"tag":191,"props":27784,"children":27785},{},[27786,27793],{"type":150,"tag":195,"props":27787,"children":27788},{},[27789],{"type":150,"tag":405,"props":27790,"children":27791},{},[27792],{"type":155,"value":24918},{"type":150,"tag":223,"props":27794,"children":27795},{"encoding":225},[27796],{"type":155,"value":24918},{"type":150,"tag":169,"props":27798,"children":27800},{"className":27799,"ariaHidden":214},[232],[27801],{"type":150,"tag":169,"props":27802,"children":27804},{"className":27803},[237],[27805,27809],{"type":150,"tag":169,"props":27806,"children":27808},{"className":27807,"style":5854},[242],[],{"type":150,"tag":169,"props":27810,"children":27812},{"className":27811,"style":27238},[248,489],[27813],{"type":155,"value":24918},{"type":155,"value":27815}," 一大，分母里的 ",{"type":150,"tag":169,"props":27817,"children":27819},{"className":27818},[177],[27820,27841],{"type":150,"tag":169,"props":27821,"children":27823},{"className":27822},[182],[27824],{"type":150,"tag":185,"props":27825,"children":27826},{"xmlns":187},[27827],{"type":150,"tag":191,"props":27828,"children":27829},{},[27830,27837],{"type":150,"tag":195,"props":27831,"children":27832},{},[27833],{"type":150,"tag":405,"props":27834,"children":27835},{},[27836],{"type":155,"value":3443},{"type":150,"tag":223,"props":27838,"children":27839},{"encoding":225},[27840],{"type":155,"value":3443},{"type":150,"tag":169,"props":27842,"children":27844},{"className":27843,"ariaHidden":214},[232],[27845],{"type":150,"tag":169,"props":27846,"children":27848},{"className":27847},[237],[27849,27853],{"type":150,"tag":169,"props":27850,"children":27852},{"className":27851,"style":5854},[242],[],{"type":150,"tag":169,"props":27854,"children":27856},{"className":27855,"style":3869},[248,489],[27857],{"type":155,"value":3443},{"type":155,"value":27859}," 就被摊薄，算术强度趋近于：",{"type":150,"tag":169,"props":27861,"children":27863},{"className":27862},[172],[27864],{"type":150,"tag":169,"props":27865,"children":27867},{"className":27866},[177],[27868,27973],{"type":150,"tag":169,"props":27869,"children":27871},{"className":27870},[182],[27872],{"type":150,"tag":185,"props":27873,"children":27874},{"xmlns":187,"display":188},[27875],{"type":150,"tag":191,"props":27876,"children":27877},{},[27878,27968],{"type":150,"tag":195,"props":27879,"children":27880},{},[27881,27885,27889,27893,27897,27901,27905,27909],{"type":150,"tag":217,"props":27882,"children":27883},{},[27884],{"type":155,"value":25141},{"type":150,"tag":417,"props":27886,"children":27887},{"stretchy":207},[27888],{"type":155,"value":972},{"type":150,"tag":405,"props":27890,"children":27891},{"mathvariant":1796},[27892],{"type":155,"value":26346},{"type":150,"tag":417,"props":27894,"children":27895},{"separator":214},[27896],{"type":155,"value":421},{"type":150,"tag":405,"props":27898,"children":27899},{},[27900],{"type":155,"value":5591},{"type":150,"tag":417,"props":27902,"children":27903},{"stretchy":207},[27904],{"type":155,"value":982},{"type":150,"tag":417,"props":27906,"children":27907},{},[27908],{"type":155,"value":1045},{"type":150,"tag":1952,"props":27910,"children":27911},{},[27912,27946],{"type":150,"tag":195,"props":27913,"children":27914},{},[27915,27919,27923,27927,27938,27942],{"type":150,"tag":411,"props":27916,"children":27917},{},[27918],{"type":155,"value":433},{"type":150,"tag":405,"props":27920,"children":27921},{},[27922],{"type":155,"value":25172},{"type":150,"tag":417,"props":27924,"children":27925},{},[27926],{"type":155,"value":5136},{"type":150,"tag":401,"props":27928,"children":27929},{},[27930,27934],{"type":150,"tag":405,"props":27931,"children":27932},{},[27933],{"type":155,"value":10046},{"type":150,"tag":217,"props":27935,"children":27936},{},[27937],{"type":155,"value":24428},{"type":150,"tag":405,"props":27939,"children":27940},{},[27941],{"type":155,"value":5559},{"type":150,"tag":405,"props":27943,"children":27944},{},[27945],{"type":155,"value":5591},{"type":150,"tag":195,"props":27947,"children":27948},{},[27949,27960,27964],{"type":150,"tag":401,"props":27950,"children":27951},{},[27952,27956],{"type":150,"tag":405,"props":27953,"children":27954},{},[27955],{"type":155,"value":10046},{"type":150,"tag":217,"props":27957,"children":27958},{},[27959],{"type":155,"value":23996},{"type":150,"tag":405,"props":27961,"children":27962},{},[27963],{"type":155,"value":5559},{"type":150,"tag":405,"props":27965,"children":27966},{},[27967],{"type":155,"value":5591},{"type":150,"tag":223,"props":27969,"children":27970},{"encoding":225},[27971],{"type":155,"value":27972},"\\text{AI}(\\infty, N) = \\frac{2P + c_{\\text{flop}} L N}{c_{\\text{kv}} L N}",{"type":150,"tag":169,"props":27974,"children":27976},{"className":27975,"ariaHidden":214},[232],[27977,28036],{"type":150,"tag":169,"props":27978,"children":27980},{"className":27979},[237],[27981,27985,27994,27999,28004,28009,28013,28018,28023,28027,28032],{"type":150,"tag":169,"props":27982,"children":27984},{"className":27983,"style":999},[242],[],{"type":150,"tag":169,"props":27986,"children":27988},{"className":27987},[248,155],[27989],{"type":150,"tag":169,"props":27990,"children":27992},{"className":27991},[248],[27993],{"type":155,"value":25141},{"type":150,"tag":169,"props":27995,"children":27997},{"className":27996},[1004],[27998],{"type":155,"value":972},{"type":150,"tag":169,"props":28000,"children":28002},{"className":28001},[248],[28003],{"type":155,"value":26346},{"type":150,"tag":169,"props":28005,"children":28007},{"className":28006},[552],[28008],{"type":155,"value":421},{"type":150,"tag":169,"props":28010,"children":28012},{"className":28011,"style":559},[558],[],{"type":150,"tag":169,"props":28014,"children":28016},{"className":28015,"style":5780},[248,489],[28017],{"type":155,"value":5591},{"type":150,"tag":169,"props":28019,"children":28021},{"className":28020},[1015],[28022],{"type":155,"value":982},{"type":150,"tag":169,"props":28024,"children":28026},{"className":28025,"style":839},[558],[],{"type":150,"tag":169,"props":28028,"children":28030},{"className":28029},[844],[28031],{"type":155,"value":1045},{"type":150,"tag":169,"props":28033,"children":28035},{"className":28034,"style":839},[558],[],{"type":150,"tag":169,"props":28037,"children":28039},{"className":28038},[237],[28040,28044],{"type":150,"tag":169,"props":28041,"children":28043},{"className":28042,"style":25304},[242],[],{"type":150,"tag":169,"props":28045,"children":28047},{"className":28046},[248],[28048,28052,28290],{"type":150,"tag":169,"props":28049,"children":28051},{"className":28050},[1004,2660],[],{"type":150,"tag":169,"props":28053,"children":28055},{"className":28054},[1952],[28056],{"type":150,"tag":169,"props":28057,"children":28059},{"className":28058},[253,254],[28060,28279],{"type":150,"tag":169,"props":28061,"children":28063},{"className":28062},[259],[28064,28274],{"type":150,"tag":169,"props":28065,"children":28067},{"className":28066,"style":23685},[264],[28068,28154,28165],{"type":150,"tag":169,"props":28069,"children":28070},{"style":8422},[28071,28075],{"type":150,"tag":169,"props":28072,"children":28074},{"className":28073,"style":8427},[274],[],{"type":150,"tag":169,"props":28076,"children":28078},{"className":28077},[248],[28079,28144,28149],{"type":150,"tag":169,"props":28080,"children":28082},{"className":28081},[248],[28083,28088],{"type":150,"tag":169,"props":28084,"children":28086},{"className":28085},[248,489],[28087],{"type":155,"value":10046},{"type":150,"tag":169,"props":28089,"children":28091},{"className":28090},[495],[28092],{"type":150,"tag":169,"props":28093,"children":28095},{"className":28094},[253,254],[28096,28133],{"type":150,"tag":169,"props":28097,"children":28099},{"className":28098},[259],[28100,28128],{"type":150,"tag":169,"props":28101,"children":28103},{"className":28102,"style":674},[264],[28104],{"type":150,"tag":169,"props":28105,"children":28106},{"style":512},[28107,28111],{"type":150,"tag":169,"props":28108,"children":28110},{"className":28109,"style":517},[274],[],{"type":150,"tag":169,"props":28112,"children":28114},{"className":28113},[522,523,524,525],[28115],{"type":150,"tag":169,"props":28116,"children":28118},{"className":28117},[248,525],[28119],{"type":150,"tag":169,"props":28120,"children":28122},{"className":28121},[248,155,525],[28123],{"type":150,"tag":169,"props":28124,"children":28126},{"className":28125},[248,525],[28127],{"type":155,"value":23996},{"type":150,"tag":169,"props":28129,"children":28131},{"className":28130},[355],[28132],{"type":155,"value":358},{"type":150,"tag":169,"props":28134,"children":28136},{"className":28135},[259],[28137],{"type":150,"tag":169,"props":28138,"children":28140},{"className":28139,"style":544},[264],[28141],{"type":150,"tag":169,"props":28142,"children":28143},{},[],{"type":150,"tag":169,"props":28145,"children":28147},{"className":28146},[248,489],[28148],{"type":155,"value":5559},{"type":150,"tag":169,"props":28150,"children":28152},{"className":28151,"style":5780},[248,489],[28153],{"type":155,"value":5591},{"type":150,"tag":169,"props":28155,"children":28156},{"style":8440},[28157,28161],{"type":150,"tag":169,"props":28158,"children":28160},{"className":28159,"style":8427},[274],[],{"type":150,"tag":169,"props":28162,"children":28164},{"className":28163,"style":2982},[2981],[],{"type":150,"tag":169,"props":28166,"children":28167},{"style":8452},[28168,28172],{"type":150,"tag":169,"props":28169,"children":28171},{"className":28170,"style":8427},[274],[],{"type":150,"tag":169,"props":28173,"children":28175},{"className":28174},[248],[28176,28181,28186,28190,28195,28199,28264,28269],{"type":150,"tag":169,"props":28177,"children":28179},{"className":28178},[248],[28180],{"type":155,"value":433},{"type":150,"tag":169,"props":28182,"children":28184},{"className":28183,"style":3869},[248,489],[28185],{"type":155,"value":25172},{"type":150,"tag":169,"props":28187,"children":28189},{"className":28188,"style":5167},[558],[],{"type":150,"tag":169,"props":28191,"children":28193},{"className":28192},[5172],[28194],{"type":155,"value":5136},{"type":150,"tag":169,"props":28196,"children":28198},{"className":28197,"style":5167},[558],[],{"type":150,"tag":169,"props":28200,"children":28202},{"className":28201},[248],[28203,28208],{"type":150,"tag":169,"props":28204,"children":28206},{"className":28205},[248,489],[28207],{"type":155,"value":10046},{"type":150,"tag":169,"props":28209,"children":28211},{"className":28210},[495],[28212],{"type":150,"tag":169,"props":28213,"children":28215},{"className":28214},[253,254],[28216,28253],{"type":150,"tag":169,"props":28217,"children":28219},{"className":28218},[259],[28220,28248],{"type":150,"tag":169,"props":28221,"children":28223},{"className":28222,"style":674},[264],[28224],{"type":150,"tag":169,"props":28225,"children":28226},{"style":512},[28227,28231],{"type":150,"tag":169,"props":28228,"children":28230},{"className":28229,"style":517},[274],[],{"type":150,"tag":169,"props":28232,"children":28234},{"className":28233},[522,523,524,525],[28235],{"type":150,"tag":169,"props":28236,"children":28238},{"className":28237},[248,525],[28239],{"type":150,"tag":169,"props":28240,"children":28242},{"className":28241},[248,155,525],[28243],{"type":150,"tag":169,"props":28244,"children":28246},{"className":28245},[248,525],[28247],{"type":155,"value":24428},{"type":150,"tag":169,"props":28249,"children":28251},{"className":28250},[355],[28252],{"type":155,"value":358},{"type":150,"tag":169,"props":28254,"children":28256},{"className":28255},[259],[28257],{"type":150,"tag":169,"props":28258,"children":28260},{"className":28259,"style":6692},[264],[28261],{"type":150,"tag":169,"props":28262,"children":28263},{},[],{"type":150,"tag":169,"props":28265,"children":28267},{"className":28266},[248,489],[28268],{"type":155,"value":5559},{"type":150,"tag":169,"props":28270,"children":28272},{"className":28271,"style":5780},[248,489],[28273],{"type":155,"value":5591},{"type":150,"tag":169,"props":28275,"children":28277},{"className":28276},[355],[28278],{"type":155,"value":358},{"type":150,"tag":169,"props":28280,"children":28282},{"className":28281},[259],[28283],{"type":150,"tag":169,"props":28284,"children":28286},{"className":28285,"style":25618},[264],[28287],{"type":150,"tag":169,"props":28288,"children":28289},{},[],{"type":150,"tag":169,"props":28291,"children":28293},{"className":28292},[1015,2660],[],{"type":150,"tag":151,"props":28295,"children":28296},{},[28297],{"type":155,"value":23289},{"type":150,"tag":22368,"props":28299,"children":28300},{},[28301,28752],{"type":150,"tag":22372,"props":28302,"children":28303},{},[28304],{"type":150,"tag":22376,"props":28305,"children":28306},{},[28307,28352,28432,28512,28592,28672],{"type":150,"tag":22380,"props":28308,"children":28309},{},[28310],{"type":150,"tag":169,"props":28311,"children":28313},{"className":28312},[177],[28314,28335],{"type":150,"tag":169,"props":28315,"children":28317},{"className":28316},[182],[28318],{"type":150,"tag":185,"props":28319,"children":28320},{"xmlns":187},[28321],{"type":150,"tag":191,"props":28322,"children":28323},{},[28324,28331],{"type":150,"tag":195,"props":28325,"children":28326},{},[28327],{"type":150,"tag":405,"props":28328,"children":28329},{},[28330],{"type":155,"value":5591},{"type":150,"tag":223,"props":28332,"children":28333},{"encoding":225},[28334],{"type":155,"value":5591},{"type":150,"tag":169,"props":28336,"children":28338},{"className":28337,"ariaHidden":214},[232],[28339],{"type":150,"tag":169,"props":28340,"children":28342},{"className":28341},[237],[28343,28347],{"type":150,"tag":169,"props":28344,"children":28346},{"className":28345,"style":5854},[242],[],{"type":150,"tag":169,"props":28348,"children":28350},{"className":28349,"style":5780},[248,489],[28351],{"type":155,"value":5591},{"type":150,"tag":22380,"props":28353,"children":28354},{},[28355],{"type":150,"tag":169,"props":28356,"children":28358},{"className":28357},[177],[28359,28389],{"type":150,"tag":169,"props":28360,"children":28362},{"className":28361},[182],[28363],{"type":150,"tag":185,"props":28364,"children":28365},{"xmlns":187},[28366],{"type":150,"tag":191,"props":28367,"children":28368},{},[28369,28384],{"type":150,"tag":195,"props":28370,"children":28371},{},[28372,28376,28380],{"type":150,"tag":405,"props":28373,"children":28374},{},[28375],{"type":155,"value":24918},{"type":150,"tag":417,"props":28377,"children":28378},{},[28379],{"type":155,"value":1045},{"type":150,"tag":411,"props":28381,"children":28382},{},[28383],{"type":155,"value":415},{"type":150,"tag":223,"props":28385,"children":28386},{"encoding":225},[28387],{"type":155,"value":28388},"B=1",{"type":150,"tag":169,"props":28390,"children":28392},{"className":28391,"ariaHidden":214},[232],[28393,28419],{"type":150,"tag":169,"props":28394,"children":28396},{"className":28395},[237],[28397,28401,28406,28410,28415],{"type":150,"tag":169,"props":28398,"children":28400},{"className":28399,"style":5854},[242],[],{"type":150,"tag":169,"props":28402,"children":28404},{"className":28403,"style":27238},[248,489],[28405],{"type":155,"value":24918},{"type":150,"tag":169,"props":28407,"children":28409},{"className":28408,"style":839},[558],[],{"type":150,"tag":169,"props":28411,"children":28413},{"className":28412},[844],[28414],{"type":155,"value":1045},{"type":150,"tag":169,"props":28416,"children":28418},{"className":28417,"style":839},[558],[],{"type":150,"tag":169,"props":28420,"children":28422},{"className":28421},[237],[28423,28427],{"type":150,"tag":169,"props":28424,"children":28426},{"className":28425,"style":5186},[242],[],{"type":150,"tag":169,"props":28428,"children":28430},{"className":28429},[248],[28431],{"type":155,"value":415},{"type":150,"tag":22380,"props":28433,"children":28434},{},[28435],{"type":150,"tag":169,"props":28436,"children":28438},{"className":28437},[177],[28439,28469],{"type":150,"tag":169,"props":28440,"children":28442},{"className":28441},[182],[28443],{"type":150,"tag":185,"props":28444,"children":28445},{"xmlns":187},[28446],{"type":150,"tag":191,"props":28447,"children":28448},{},[28449,28464],{"type":150,"tag":195,"props":28450,"children":28451},{},[28452,28456,28460],{"type":150,"tag":405,"props":28453,"children":28454},{},[28455],{"type":155,"value":24918},{"type":150,"tag":417,"props":28457,"children":28458},{},[28459],{"type":155,"value":1045},{"type":150,"tag":411,"props":28461,"children":28462},{},[28463],{"type":155,"value":9389},{"type":150,"tag":223,"props":28465,"children":28466},{"encoding":225},[28467],{"type":155,"value":28468},"B=8",{"type":150,"tag":169,"props":28470,"children":28472},{"className":28471,"ariaHidden":214},[232],[28473,28499],{"type":150,"tag":169,"props":28474,"children":28476},{"className":28475},[237],[28477,28481,28486,28490,28495],{"type":150,"tag":169,"props":28478,"children":28480},{"className":28479,"style":5854},[242],[],{"type":150,"tag":169,"props":28482,"children":28484},{"className":28483,"style":27238},[248,489],[28485],{"type":155,"value":24918},{"type":150,"tag":169,"props":28487,"children":28489},{"className":28488,"style":839},[558],[],{"type":150,"tag":169,"props":28491,"children":28493},{"className":28492},[844],[28494],{"type":155,"value":1045},{"type":150,"tag":169,"props":28496,"children":28498},{"className":28497,"style":839},[558],[],{"type":150,"tag":169,"props":28500,"children":28502},{"className":28501},[237],[28503,28507],{"type":150,"tag":169,"props":28504,"children":28506},{"className":28505,"style":5186},[242],[],{"type":150,"tag":169,"props":28508,"children":28510},{"className":28509},[248],[28511],{"type":155,"value":9389},{"type":150,"tag":22380,"props":28513,"children":28514},{},[28515],{"type":150,"tag":169,"props":28516,"children":28518},{"className":28517},[177],[28519,28549],{"type":150,"tag":169,"props":28520,"children":28522},{"className":28521},[182],[28523],{"type":150,"tag":185,"props":28524,"children":28525},{"xmlns":187},[28526],{"type":150,"tag":191,"props":28527,"children":28528},{},[28529,28544],{"type":150,"tag":195,"props":28530,"children":28531},{},[28532,28536,28540],{"type":150,"tag":405,"props":28533,"children":28534},{},[28535],{"type":155,"value":24918},{"type":150,"tag":417,"props":28537,"children":28538},{},[28539],{"type":155,"value":1045},{"type":150,"tag":411,"props":28541,"children":28542},{},[28543],{"type":155,"value":20836},{"type":150,"tag":223,"props":28545,"children":28546},{"encoding":225},[28547],{"type":155,"value":28548},"B=64",{"type":150,"tag":169,"props":28550,"children":28552},{"className":28551,"ariaHidden":214},[232],[28553,28579],{"type":150,"tag":169,"props":28554,"children":28556},{"className":28555},[237],[28557,28561,28566,28570,28575],{"type":150,"tag":169,"props":28558,"children":28560},{"className":28559,"style":5854},[242],[],{"type":150,"tag":169,"props":28562,"children":28564},{"className":28563,"style":27238},[248,489],[28565],{"type":155,"value":24918},{"type":150,"tag":169,"props":28567,"children":28569},{"className":28568,"style":839},[558],[],{"type":150,"tag":169,"props":28571,"children":28573},{"className":28572},[844],[28574],{"type":155,"value":1045},{"type":150,"tag":169,"props":28576,"children":28578},{"className":28577,"style":839},[558],[],{"type":150,"tag":169,"props":28580,"children":28582},{"className":28581},[237],[28583,28587],{"type":150,"tag":169,"props":28584,"children":28586},{"className":28585,"style":5186},[242],[],{"type":150,"tag":169,"props":28588,"children":28590},{"className":28589},[248],[28591],{"type":155,"value":20836},{"type":150,"tag":22380,"props":28593,"children":28594},{},[28595],{"type":150,"tag":169,"props":28596,"children":28598},{"className":28597},[177],[28599,28629],{"type":150,"tag":169,"props":28600,"children":28602},{"className":28601},[182],[28603],{"type":150,"tag":185,"props":28604,"children":28605},{"xmlns":187},[28606],{"type":150,"tag":191,"props":28607,"children":28608},{},[28609,28624],{"type":150,"tag":195,"props":28610,"children":28611},{},[28612,28616,28620],{"type":150,"tag":405,"props":28613,"children":28614},{},[28615],{"type":155,"value":24918},{"type":150,"tag":417,"props":28617,"children":28618},{},[28619],{"type":155,"value":1045},{"type":150,"tag":411,"props":28621,"children":28622},{},[28623],{"type":155,"value":20981},{"type":150,"tag":223,"props":28625,"children":28626},{"encoding":225},[28627],{"type":155,"value":28628},"B=512",{"type":150,"tag":169,"props":28630,"children":28632},{"className":28631,"ariaHidden":214},[232],[28633,28659],{"type":150,"tag":169,"props":28634,"children":28636},{"className":28635},[237],[28637,28641,28646,28650,28655],{"type":150,"tag":169,"props":28638,"children":28640},{"className":28639,"style":5854},[242],[],{"type":150,"tag":169,"props":28642,"children":28644},{"className":28643,"style":27238},[248,489],[28645],{"type":155,"value":24918},{"type":150,"tag":169,"props":28647,"children":28649},{"className":28648,"style":839},[558],[],{"type":150,"tag":169,"props":28651,"children":28653},{"className":28652},[844],[28654],{"type":155,"value":1045},{"type":150,"tag":169,"props":28656,"children":28658},{"className":28657,"style":839},[558],[],{"type":150,"tag":169,"props":28660,"children":28662},{"className":28661},[237],[28663,28667],{"type":150,"tag":169,"props":28664,"children":28666},{"className":28665,"style":5186},[242],[],{"type":150,"tag":169,"props":28668,"children":28670},{"className":28669},[248],[28671],{"type":155,"value":20981},{"type":150,"tag":22380,"props":28673,"children":28674},{},[28675],{"type":150,"tag":169,"props":28676,"children":28678},{"className":28677},[177],[28679,28709],{"type":150,"tag":169,"props":28680,"children":28682},{"className":28681},[182],[28683],{"type":150,"tag":185,"props":28684,"children":28685},{"xmlns":187},[28686],{"type":150,"tag":191,"props":28687,"children":28688},{},[28689,28704],{"type":150,"tag":195,"props":28690,"children":28691},{},[28692,28696,28700],{"type":150,"tag":405,"props":28693,"children":28694},{},[28695],{"type":155,"value":24918},{"type":150,"tag":417,"props":28697,"children":28698},{},[28699],{"type":155,"value":12954},{"type":150,"tag":405,"props":28701,"children":28702},{"mathvariant":1796},[28703],{"type":155,"value":26346},{"type":150,"tag":223,"props":28705,"children":28706},{"encoding":225},[28707],{"type":155,"value":28708},"B \\to \\infty",{"type":150,"tag":169,"props":28710,"children":28712},{"className":28711,"ariaHidden":214},[232],[28713,28739],{"type":150,"tag":169,"props":28714,"children":28716},{"className":28715},[237],[28717,28721,28726,28730,28735],{"type":150,"tag":169,"props":28718,"children":28720},{"className":28719,"style":5854},[242],[],{"type":150,"tag":169,"props":28722,"children":28724},{"className":28723,"style":27238},[248,489],[28725],{"type":155,"value":24918},{"type":150,"tag":169,"props":28727,"children":28729},{"className":28728,"style":839},[558],[],{"type":150,"tag":169,"props":28731,"children":28733},{"className":28732},[844],[28734],{"type":155,"value":12954},{"type":150,"tag":169,"props":28736,"children":28738},{"className":28737,"style":839},[558],[],{"type":150,"tag":169,"props":28740,"children":28742},{"className":28741},[237],[28743,28747],{"type":150,"tag":169,"props":28744,"children":28746},{"className":28745,"style":1094},[242],[],{"type":150,"tag":169,"props":28748,"children":28750},{"className":28749},[248],[28751],{"type":155,"value":26346},{"type":150,"tag":22394,"props":28753,"children":28754},{},[28755,28792,28823,28854],{"type":150,"tag":22376,"props":28756,"children":28757},{},[28758,28762,28766,28771,28776,28784],{"type":150,"tag":22401,"props":28759,"children":28760},{},[28761],{"type":155,"value":27081},{"type":150,"tag":22401,"props":28763,"children":28764},{},[28765],{"type":155,"value":27086},{"type":150,"tag":22401,"props":28767,"children":28768},{},[28769],{"type":155,"value":28770},"20.0",{"type":150,"tag":22401,"props":28772,"children":28773},{},[28774],{"type":155,"value":28775},"118",{"type":150,"tag":22401,"props":28777,"children":28778},{},[28779],{"type":150,"tag":158,"props":28780,"children":28781},{},[28782],{"type":155,"value":28783},"303",{"type":150,"tag":22401,"props":28785,"children":28786},{},[28787],{"type":150,"tag":158,"props":28788,"children":28789},{},[28790],{"type":155,"value":28791},"390",{"type":150,"tag":22376,"props":28793,"children":28794},{},[28795,28799,28803,28808,28813,28818],{"type":150,"tag":22401,"props":28796,"children":28797},{},[28798],{"type":155,"value":27109},{"type":150,"tag":22401,"props":28800,"children":28801},{},[28802],{"type":155,"value":27114},{"type":150,"tag":22401,"props":28804,"children":28805},{},[28806],{"type":155,"value":28807},"78.5",{"type":150,"tag":22401,"props":28809,"children":28810},{},[28811],{"type":155,"value":28812},"202",{"type":150,"tag":22401,"props":28814,"children":28815},{},[28816],{"type":155,"value":28817},"251",{"type":150,"tag":22401,"props":28819,"children":28820},{},[28821],{"type":155,"value":28822},"260",{"type":150,"tag":22376,"props":28824,"children":28825},{},[28826,28830,28834,28839,28844,28849],{"type":150,"tag":22401,"props":28827,"children":28828},{},[28829],{"type":155,"value":27136},{"type":150,"tag":22401,"props":28831,"children":28832},{},[28833],{"type":155,"value":27141},{"type":150,"tag":22401,"props":28835,"children":28836},{},[28837],{"type":155,"value":28838},"156",{"type":150,"tag":22401,"props":28840,"children":28841},{},[28842],{"type":155,"value":28843},"230",{"type":150,"tag":22401,"props":28845,"children":28846},{},[28847],{"type":155,"value":28848},"244",{"type":150,"tag":22401,"props":28850,"children":28851},{},[28852],{"type":155,"value":28853},"246",{"type":150,"tag":22376,"props":28855,"children":28856},{},[28857,28861,28865,28870,28875,28879],{"type":150,"tag":22401,"props":28858,"children":28859},{},[28860],{"type":155,"value":27163},{"type":150,"tag":22401,"props":28862,"children":28863},{},[28864],{"type":155,"value":27168},{"type":150,"tag":22401,"props":28866,"children":28867},{},[28868],{"type":155,"value":28869},"226",{"type":150,"tag":22401,"props":28871,"children":28872},{},[28873],{"type":155,"value":28874},"240",{"type":150,"tag":22401,"props":28876,"children":28877},{},[28878],{"type":155,"value":26415},{"type":150,"tag":22401,"props":28880,"children":28881},{},[28882],{"type":155,"value":26415},{"type":150,"tag":151,"props":28884,"children":28885},{},[28886],{"type":155,"value":28887},"（加粗的是越过 295 的格子。）",{"type":150,"tag":151,"props":28889,"children":28890},{},[28891],{"type":155,"value":28892},"于是结论要分区间说：",{"type":150,"tag":23833,"props":28894,"children":28895},{},[28896,28906],{"type":150,"tag":23837,"props":28897,"children":28898},{},[28899,28904],{"type":150,"tag":158,"props":28900,"children":28901},{},[28902],{"type":155,"value":28903},"短 context（4k 这一档）",{"type":155,"value":28905},"：batch 开得够大，decode 会翻到 compute bound。这时候那 4.25 倍的计算量是真的要还的。",{"type":150,"tag":23837,"props":28907,"children":28908},{},[28909,28914],{"type":150,"tag":158,"props":28910,"children":28911},{},[28912],{"type":155,"value":28913},"32k 及以上",{"type":155,"value":28915},"：即使 batch 开到无穷，算术强度也够不到 295。decode 始终 memory bound，那 4.25 倍照样白加。",{"type":150,"tag":151,"props":28917,"children":28918},{},[28919,28921,28963,28965,29093,29095,29146,29148,29378],{"type":155,"value":28920},"原因在渐近值里写得清楚。",{"type":150,"tag":169,"props":28922,"children":28924},{"className":28923},[177],[28925,28946],{"type":150,"tag":169,"props":28926,"children":28928},{"className":28927},[182],[28929],{"type":150,"tag":185,"props":28930,"children":28931},{"xmlns":187},[28932],{"type":150,"tag":191,"props":28933,"children":28934},{},[28935,28942],{"type":150,"tag":195,"props":28936,"children":28937},{},[28938],{"type":150,"tag":405,"props":28939,"children":28940},{},[28941],{"type":155,"value":5591},{"type":150,"tag":223,"props":28943,"children":28944},{"encoding":225},[28945],{"type":155,"value":5591},{"type":150,"tag":169,"props":28947,"children":28949},{"className":28948,"ariaHidden":214},[232],[28950],{"type":150,"tag":169,"props":28951,"children":28953},{"className":28952},[237],[28954,28958],{"type":150,"tag":169,"props":28955,"children":28957},{"className":28956,"style":5854},[242],[],{"type":150,"tag":169,"props":28959,"children":28961},{"className":28960,"style":5780},[248,489],[28962],{"type":155,"value":5591},{"type":155,"value":28964}," 一大，",{"type":150,"tag":169,"props":28966,"children":28968},{"className":28967},[177],[28969,29006],{"type":150,"tag":169,"props":28970,"children":28972},{"className":28971},[182],[28973],{"type":150,"tag":185,"props":28974,"children":28975},{"xmlns":187},[28976],{"type":150,"tag":191,"props":28977,"children":28978},{},[28979,29001],{"type":150,"tag":195,"props":28980,"children":28981},{},[28982,28993,28997],{"type":150,"tag":401,"props":28983,"children":28984},{},[28985,28989],{"type":150,"tag":405,"props":28986,"children":28987},{},[28988],{"type":155,"value":10046},{"type":150,"tag":217,"props":28990,"children":28991},{},[28992],{"type":155,"value":24428},{"type":150,"tag":405,"props":28994,"children":28995},{},[28996],{"type":155,"value":5559},{"type":150,"tag":405,"props":28998,"children":28999},{},[29000],{"type":155,"value":5591},{"type":150,"tag":223,"props":29002,"children":29003},{"encoding":225},[29004],{"type":155,"value":29005},"c_{\\text{flop}} L N",{"type":150,"tag":169,"props":29007,"children":29009},{"className":29008,"ariaHidden":214},[232],[29010],{"type":150,"tag":169,"props":29011,"children":29013},{"className":29012},[237],[29014,29018,29083,29088],{"type":150,"tag":169,"props":29015,"children":29017},{"className":29016,"style":6629},[242],[],{"type":150,"tag":169,"props":29019,"children":29021},{"className":29020},[248],[29022,29027],{"type":150,"tag":169,"props":29023,"children":29025},{"className":29024},[248,489],[29026],{"type":155,"value":10046},{"type":150,"tag":169,"props":29028,"children":29030},{"className":29029},[495],[29031],{"type":150,"tag":169,"props":29032,"children":29034},{"className":29033},[253,254],[29035,29072],{"type":150,"tag":169,"props":29036,"children":29038},{"className":29037},[259],[29039,29067],{"type":150,"tag":169,"props":29040,"children":29042},{"className":29041,"style":674},[264],[29043],{"type":150,"tag":169,"props":29044,"children":29045},{"style":512},[29046,29050],{"type":150,"tag":169,"props":29047,"children":29049},{"className":29048,"style":517},[274],[],{"type":150,"tag":169,"props":29051,"children":29053},{"className":29052},[522,523,524,525],[29054],{"type":150,"tag":169,"props":29055,"children":29057},{"className":29056},[248,525],[29058],{"type":150,"tag":169,"props":29059,"children":29061},{"className":29060},[248,155,525],[29062],{"type":150,"tag":169,"props":29063,"children":29065},{"className":29064},[248,525],[29066],{"type":155,"value":24428},{"type":150,"tag":169,"props":29068,"children":29070},{"className":29069},[355],[29071],{"type":155,"value":358},{"type":150,"tag":169,"props":29073,"children":29075},{"className":29074},[259],[29076],{"type":150,"tag":169,"props":29077,"children":29079},{"className":29078,"style":6692},[264],[29080],{"type":150,"tag":169,"props":29081,"children":29082},{},[],{"type":150,"tag":169,"props":29084,"children":29086},{"className":29085},[248,489],[29087],{"type":155,"value":5559},{"type":150,"tag":169,"props":29089,"children":29091},{"className":29090,"style":5780},[248,489],[29092],{"type":155,"value":5591},{"type":155,"value":29094}," 就压过 ",{"type":150,"tag":169,"props":29096,"children":29098},{"className":29097},[177],[29099,29124],{"type":150,"tag":169,"props":29100,"children":29102},{"className":29101},[182],[29103],{"type":150,"tag":185,"props":29104,"children":29105},{"xmlns":187},[29106],{"type":150,"tag":191,"props":29107,"children":29108},{},[29109,29120],{"type":150,"tag":195,"props":29110,"children":29111},{},[29112,29116],{"type":150,"tag":411,"props":29113,"children":29114},{},[29115],{"type":155,"value":433},{"type":150,"tag":405,"props":29117,"children":29118},{},[29119],{"type":155,"value":25172},{"type":150,"tag":223,"props":29121,"children":29122},{"encoding":225},[29123],{"type":155,"value":25661},{"type":150,"tag":169,"props":29125,"children":29127},{"className":29126,"ariaHidden":214},[232],[29128],{"type":150,"tag":169,"props":29129,"children":29131},{"className":29130},[237],[29132,29136,29141],{"type":150,"tag":169,"props":29133,"children":29135},{"className":29134,"style":5854},[242],[],{"type":150,"tag":169,"props":29137,"children":29139},{"className":29138},[248],[29140],{"type":155,"value":433},{"type":150,"tag":169,"props":29142,"children":29144},{"className":29143,"style":3869},[248,489],[29145],{"type":155,"value":25172},{"type":155,"value":29147},"，权重项被稀释掉，剩下的比值就是 ",{"type":150,"tag":169,"props":29149,"children":29151},{"className":29150},[177],[29152,29204],{"type":150,"tag":169,"props":29153,"children":29155},{"className":29154},[182],[29156],{"type":150,"tag":185,"props":29157,"children":29158},{"xmlns":187},[29159],{"type":150,"tag":191,"props":29160,"children":29161},{},[29162,29199],{"type":150,"tag":195,"props":29163,"children":29164},{},[29165,29176,29180,29191,29195],{"type":150,"tag":401,"props":29166,"children":29167},{},[29168,29172],{"type":150,"tag":405,"props":29169,"children":29170},{},[29171],{"type":155,"value":10046},{"type":150,"tag":217,"props":29173,"children":29174},{},[29175],{"type":155,"value":24428},{"type":150,"tag":405,"props":29177,"children":29178},{"mathvariant":1796},[29179],{"type":155,"value":4914},{"type":150,"tag":401,"props":29181,"children":29182},{},[29183,29187],{"type":150,"tag":405,"props":29184,"children":29185},{},[29186],{"type":155,"value":10046},{"type":150,"tag":217,"props":29188,"children":29189},{},[29190],{"type":155,"value":23996},{"type":150,"tag":417,"props":29192,"children":29193},{},[29194],{"type":155,"value":1045},{"type":150,"tag":411,"props":29196,"children":29197},{},[29198],{"type":155,"value":26415},{"type":150,"tag":223,"props":29200,"children":29201},{"encoding":225},[29202],{"type":155,"value":29203},"c_{\\text{flop}}\u002Fc_{\\text{kv}} = 242",{"type":150,"tag":169,"props":29205,"children":29207},{"className":29206,"ariaHidden":214},[232],[29208,29365],{"type":150,"tag":169,"props":29209,"children":29211},{"className":29210},[237],[29212,29217,29282,29287,29352,29356,29361],{"type":150,"tag":169,"props":29213,"children":29216},{"className":29214,"style":29215},[242],"height:1.0361em;vertical-align:-0.2861em;",[],{"type":150,"tag":169,"props":29218,"children":29220},{"className":29219},[248],[29221,29226],{"type":150,"tag":169,"props":29222,"children":29224},{"className":29223},[248,489],[29225],{"type":155,"value":10046},{"type":150,"tag":169,"props":29227,"children":29229},{"className":29228},[495],[29230],{"type":150,"tag":169,"props":29231,"children":29233},{"className":29232},[253,254],[29234,29271],{"type":150,"tag":169,"props":29235,"children":29237},{"className":29236},[259],[29238,29266],{"type":150,"tag":169,"props":29239,"children":29241},{"className":29240,"style":674},[264],[29242],{"type":150,"tag":169,"props":29243,"children":29244},{"style":512},[29245,29249],{"type":150,"tag":169,"props":29246,"children":29248},{"className":29247,"style":517},[274],[],{"type":150,"tag":169,"props":29250,"children":29252},{"className":29251},[522,523,524,525],[29253],{"type":150,"tag":169,"props":29254,"children":29256},{"className":29255},[248,525],[29257],{"type":150,"tag":169,"props":29258,"children":29260},{"className":29259},[248,155,525],[29261],{"type":150,"tag":169,"props":29262,"children":29264},{"className":29263},[248,525],[29265],{"type":155,"value":24428},{"type":150,"tag":169,"props":29267,"children":29269},{"className":29268},[355],[29270],{"type":155,"value":358},{"type":150,"tag":169,"props":29272,"children":29274},{"className":29273},[259],[29275],{"type":150,"tag":169,"props":29276,"children":29278},{"className":29277,"style":6692},[264],[29279],{"type":150,"tag":169,"props":29280,"children":29281},{},[],{"type":150,"tag":169,"props":29283,"children":29285},{"className":29284},[248],[29286],{"type":155,"value":4914},{"type":150,"tag":169,"props":29288,"children":29290},{"className":29289},[248],[29291,29296],{"type":150,"tag":169,"props":29292,"children":29294},{"className":29293},[248,489],[29295],{"type":155,"value":10046},{"type":150,"tag":169,"props":29297,"children":29299},{"className":29298},[495],[29300],{"type":150,"tag":169,"props":29301,"children":29303},{"className":29302},[253,254],[29304,29341],{"type":150,"tag":169,"props":29305,"children":29307},{"className":29306},[259],[29308,29336],{"type":150,"tag":169,"props":29309,"children":29311},{"className":29310,"style":674},[264],[29312],{"type":150,"tag":169,"props":29313,"children":29314},{"style":512},[29315,29319],{"type":150,"tag":169,"props":29316,"children":29318},{"className":29317,"style":517},[274],[],{"type":150,"tag":169,"props":29320,"children":29322},{"className":29321},[522,523,524,525],[29323],{"type":150,"tag":169,"props":29324,"children":29326},{"className":29325},[248,525],[29327],{"type":150,"tag":169,"props":29328,"children":29330},{"className":29329},[248,155,525],[29331],{"type":150,"tag":169,"props":29332,"children":29334},{"className":29333},[248,525],[29335],{"type":155,"value":23996},{"type":150,"tag":169,"props":29337,"children":29339},{"className":29338},[355],[29340],{"type":155,"value":358},{"type":150,"tag":169,"props":29342,"children":29344},{"className":29343},[259],[29345],{"type":150,"tag":169,"props":29346,"children":29348},{"className":29347,"style":544},[264],[29349],{"type":150,"tag":169,"props":29350,"children":29351},{},[],{"type":150,"tag":169,"props":29353,"children":29355},{"className":29354,"style":839},[558],[],{"type":150,"tag":169,"props":29357,"children":29359},{"className":29358},[844],[29360],{"type":155,"value":1045},{"type":150,"tag":169,"props":29362,"children":29364},{"className":29363,"style":839},[558],[],{"type":150,"tag":169,"props":29366,"children":29368},{"className":29367},[237],[29369,29373],{"type":150,"tag":169,"props":29370,"children":29372},{"className":29371,"style":5186},[242],[],{"type":150,"tag":169,"props":29374,"children":29376},{"className":29375},[248],[29377],{"type":155,"value":26415},{"type":155,"value":29379},"，跟 batch 无关。",{"type":150,"tag":151,"props":29381,"children":29382},{},[29383,29385],{"type":155,"value":29384},"所以 batch 只在短 context 上有效，而短 context 恰恰是 MLA 收益最小的区间。前面那张加速比表里，4k 只有 1.4×。两件事碰在一起，这个取舍就更清楚了：",{"type":150,"tag":158,"props":29386,"children":29387},{},[29388],{"type":155,"value":29389},"MLA 在自己最有价值的长 context 区间永远是 memory bound，那笔计算量从头到尾都不用还。",{"type":150,"tag":151,"props":29391,"children":29392},{},[29393,29395,29400],{"type":155,"value":29394},"长 context 下 batch 也开不大，还有另一个原因：KV Cache 装不下。MLA 在 128k 时每条序列 9.06 GB，32 条就是 290 GB，早就超出单卡。而同样条件下的 MHA 是 515 GB，",{"type":150,"tag":158,"props":29396,"children":29397},{},[29398],{"type":155,"value":29399},"一条序列都放不下",{"type":155,"value":904},{"type":150,"tag":151,"props":29402,"children":29403},{},[29404],{"type":155,"value":29405},"所以 MHA 在长 context 不是一个\"慢\"的方案，是一个\"跑不起来\"的方案。",{"type":150,"tag":151,"props":29407,"children":29408},{},[29409],{"type":155,"value":29410},"那\"划算\"的刻度在哪里？MLA 省的是字节，而字节由权重和 KV 两部分组成。KV 读取追平权重的位置：",{"type":150,"tag":169,"props":29412,"children":29414},{"className":29413},[172],[29415],{"type":150,"tag":169,"props":29416,"children":29418},{"className":29417},[177],[29419,29496],{"type":150,"tag":169,"props":29420,"children":29422},{"className":29421},[182],[29423],{"type":150,"tag":185,"props":29424,"children":29425},{"xmlns":187,"display":188},[29426],{"type":150,"tag":191,"props":29427,"children":29428},{},[29429,29491],{"type":150,"tag":195,"props":29430,"children":29431},{},[29432,29437,29442,29447,29451,29455,29459,29463,29468,29473,29477,29481,29486],{"type":150,"tag":217,"props":29433,"children":29434},{},[29435],{"type":155,"value":29436},"MHA： ",{"type":150,"tag":411,"props":29438,"children":29439},{},[29440],{"type":155,"value":29441},"3.93",{"type":150,"tag":217,"props":29443,"children":29444},{},[29445],{"type":155,"value":29446}," MB",{"type":150,"tag":417,"props":29448,"children":29449},{},[29450],{"type":155,"value":5554},{"type":150,"tag":405,"props":29452,"children":29453},{},[29454],{"type":155,"value":5591},{"type":150,"tag":417,"props":29456,"children":29457},{},[29458],{"type":155,"value":1045},{"type":150,"tag":411,"props":29460,"children":29461},{},[29462],{"type":155,"value":24940},{"type":150,"tag":217,"props":29464,"children":29465},{},[29466],{"type":155,"value":29467}," GB",{"type":150,"tag":417,"props":29469,"children":29470},{},[29471],{"type":155,"value":29472},"⇒",{"type":150,"tag":405,"props":29474,"children":29475},{},[29476],{"type":155,"value":5591},{"type":150,"tag":417,"props":29478,"children":29479},{},[29480],{"type":155,"value":6050},{"type":150,"tag":411,"props":29482,"children":29483},{},[29484],{"type":155,"value":29485},"1.1",{"type":150,"tag":217,"props":29487,"children":29488},{},[29489],{"type":155,"value":29490}," 万",{"type":150,"tag":223,"props":29492,"children":29493},{"encoding":225},[29494],{"type":155,"value":29495},"\\text{MHA：}\\ 3.93\\ \\text{MB} \\times N = 42\\ \\text{GB} \\Rightarrow N \\approx 1.1\\ \\text{万}",{"type":150,"tag":169,"props":29497,"children":29499},{"className":29498,"ariaHidden":214},[232],[29500,29560,29586,29627,29653],{"type":150,"tag":169,"props":29501,"children":29503},{"className":29502},[237],[29504,29508,29522,29527,29532,29537,29547,29551,29556],{"type":150,"tag":169,"props":29505,"children":29507},{"className":29506,"style":5644},[242],[],{"type":150,"tag":169,"props":29509,"children":29511},{"className":29510},[248,155],[29512,29517],{"type":150,"tag":169,"props":29513,"children":29515},{"className":29514},[248],[29516],{"type":155,"value":22405},{"type":150,"tag":169,"props":29518,"children":29520},{"className":29519},[248,297],[29521],{"type":155,"value":11339},{"type":150,"tag":169,"props":29523,"children":29525},{"className":29524},[558],[29526],{"type":155,"value":8763},{"type":150,"tag":169,"props":29528,"children":29530},{"className":29529},[248],[29531],{"type":155,"value":29441},{"type":150,"tag":169,"props":29533,"children":29535},{"className":29534},[558],[29536],{"type":155,"value":8763},{"type":150,"tag":169,"props":29538,"children":29540},{"className":29539},[248,155],[29541],{"type":150,"tag":169,"props":29542,"children":29544},{"className":29543},[248],[29545],{"type":155,"value":29546},"MB",{"type":150,"tag":169,"props":29548,"children":29550},{"className":29549,"style":5167},[558],[],{"type":150,"tag":169,"props":29552,"children":29554},{"className":29553},[5172],[29555],{"type":155,"value":5554},{"type":150,"tag":169,"props":29557,"children":29559},{"className":29558,"style":5167},[558],[],{"type":150,"tag":169,"props":29561,"children":29563},{"className":29562},[237],[29564,29568,29573,29577,29582],{"type":150,"tag":169,"props":29565,"children":29567},{"className":29566,"style":5854},[242],[],{"type":150,"tag":169,"props":29569,"children":29571},{"className":29570,"style":5780},[248,489],[29572],{"type":155,"value":5591},{"type":150,"tag":169,"props":29574,"children":29576},{"className":29575,"style":839},[558],[],{"type":150,"tag":169,"props":29578,"children":29580},{"className":29579},[844],[29581],{"type":155,"value":1045},{"type":150,"tag":169,"props":29583,"children":29585},{"className":29584,"style":839},[558],[],{"type":150,"tag":169,"props":29587,"children":29589},{"className":29588},[237],[29590,29594,29599,29604,29614,29618,29623],{"type":150,"tag":169,"props":29591,"children":29593},{"className":29592,"style":5854},[242],[],{"type":150,"tag":169,"props":29595,"children":29597},{"className":29596},[248],[29598],{"type":155,"value":24940},{"type":150,"tag":169,"props":29600,"children":29602},{"className":29601},[558],[29603],{"type":155,"value":8763},{"type":150,"tag":169,"props":29605,"children":29607},{"className":29606},[248,155],[29608],{"type":150,"tag":169,"props":29609,"children":29611},{"className":29610},[248],[29612],{"type":155,"value":29613},"GB",{"type":150,"tag":169,"props":29615,"children":29617},{"className":29616,"style":839},[558],[],{"type":150,"tag":169,"props":29619,"children":29621},{"className":29620},[844],[29622],{"type":155,"value":29472},{"type":150,"tag":169,"props":29624,"children":29626},{"className":29625,"style":839},[558],[],{"type":150,"tag":169,"props":29628,"children":29630},{"className":29629},[237],[29631,29635,29640,29644,29649],{"type":150,"tag":169,"props":29632,"children":29634},{"className":29633,"style":5854},[242],[],{"type":150,"tag":169,"props":29636,"children":29638},{"className":29637,"style":5780},[248,489],[29639],{"type":155,"value":5591},{"type":150,"tag":169,"props":29641,"children":29643},{"className":29642,"style":839},[558],[],{"type":150,"tag":169,"props":29645,"children":29647},{"className":29646},[844],[29648],{"type":155,"value":6050},{"type":150,"tag":169,"props":29650,"children":29652},{"className":29651,"style":839},[558],[],{"type":150,"tag":169,"props":29654,"children":29656},{"className":29655},[237],[29657,29661,29666,29671],{"type":150,"tag":169,"props":29658,"children":29660},{"className":29659,"style":5854},[242],[],{"type":150,"tag":169,"props":29662,"children":29664},{"className":29663},[248],[29665],{"type":155,"value":29485},{"type":150,"tag":169,"props":29667,"children":29669},{"className":29668},[558],[29670],{"type":155,"value":8763},{"type":150,"tag":169,"props":29672,"children":29674},{"className":29673},[248,155],[29675],{"type":150,"tag":169,"props":29676,"children":29678},{"className":29677},[248,297],[29679],{"type":155,"value":29680},"万",{"type":150,"tag":169,"props":29682,"children":29684},{"className":29683},[172],[29685],{"type":150,"tag":169,"props":29686,"children":29688},{"className":29687},[177],[29689,29763],{"type":150,"tag":169,"props":29690,"children":29692},{"className":29691},[182],[29693],{"type":150,"tag":185,"props":29694,"children":29695},{"xmlns":187,"display":188},[29696],{"type":150,"tag":191,"props":29697,"children":29698},{},[29699,29758],{"type":150,"tag":195,"props":29700,"children":29701},{},[29702,29707,29712,29717,29721,29725,29729,29733,29737,29741,29745,29749,29754],{"type":150,"tag":217,"props":29703,"children":29704},{},[29705],{"type":155,"value":29706},"MLA： ",{"type":150,"tag":411,"props":29708,"children":29709},{},[29710],{"type":155,"value":29711},"69.1",{"type":150,"tag":217,"props":29713,"children":29714},{},[29715],{"type":155,"value":29716}," KB",{"type":150,"tag":417,"props":29718,"children":29719},{},[29720],{"type":155,"value":5554},{"type":150,"tag":405,"props":29722,"children":29723},{},[29724],{"type":155,"value":5591},{"type":150,"tag":417,"props":29726,"children":29727},{},[29728],{"type":155,"value":1045},{"type":150,"tag":411,"props":29730,"children":29731},{},[29732],{"type":155,"value":24940},{"type":150,"tag":217,"props":29734,"children":29735},{},[29736],{"type":155,"value":29467},{"type":150,"tag":417,"props":29738,"children":29739},{},[29740],{"type":155,"value":29472},{"type":150,"tag":405,"props":29742,"children":29743},{},[29744],{"type":155,"value":5591},{"type":150,"tag":417,"props":29746,"children":29747},{},[29748],{"type":155,"value":6050},{"type":150,"tag":411,"props":29750,"children":29751},{},[29752],{"type":155,"value":29753},"61",{"type":150,"tag":217,"props":29755,"children":29756},{},[29757],{"type":155,"value":29490},{"type":150,"tag":223,"props":29759,"children":29760},{"encoding":225},[29761],{"type":155,"value":29762},"\\text{MLA：}\\ 69.1\\ \\text{KB} \\times N = 42\\ \\text{GB} \\Rightarrow N \\approx 61\\ \\text{万}",{"type":150,"tag":169,"props":29764,"children":29766},{"className":29765,"ariaHidden":214},[232],[29767,29827,29853,29893,29919],{"type":150,"tag":169,"props":29768,"children":29770},{"className":29769},[237],[29771,29775,29789,29794,29799,29804,29814,29818,29823],{"type":150,"tag":169,"props":29772,"children":29774},{"className":29773,"style":5644},[242],[],{"type":150,"tag":169,"props":29776,"children":29778},{"className":29777},[248,155],[29779,29784],{"type":150,"tag":169,"props":29780,"children":29782},{"className":29781},[248],[29783],{"type":155,"value":13},{"type":150,"tag":169,"props":29785,"children":29787},{"className":29786},[248,297],[29788],{"type":155,"value":11339},{"type":150,"tag":169,"props":29790,"children":29792},{"className":29791},[558],[29793],{"type":155,"value":8763},{"type":150,"tag":169,"props":29795,"children":29797},{"className":29796},[248],[29798],{"type":155,"value":29711},{"type":150,"tag":169,"props":29800,"children":29802},{"className":29801},[558],[29803],{"type":155,"value":8763},{"type":150,"tag":169,"props":29805,"children":29807},{"className":29806},[248,155],[29808],{"type":150,"tag":169,"props":29809,"children":29811},{"className":29810},[248],[29812],{"type":155,"value":29813},"KB",{"type":150,"tag":169,"props":29815,"children":29817},{"className":29816,"style":5167},[558],[],{"type":150,"tag":169,"props":29819,"children":29821},{"className":29820},[5172],[29822],{"type":155,"value":5554},{"type":150,"tag":169,"props":29824,"children":29826},{"className":29825,"style":5167},[558],[],{"type":150,"tag":169,"props":29828,"children":29830},{"className":29829},[237],[29831,29835,29840,29844,29849],{"type":150,"tag":169,"props":29832,"children":29834},{"className":29833,"style":5854},[242],[],{"type":150,"tag":169,"props":29836,"children":29838},{"className":29837,"style":5780},[248,489],[29839],{"type":155,"value":5591},{"type":150,"tag":169,"props":29841,"children":29843},{"className":29842,"style":839},[558],[],{"type":150,"tag":169,"props":29845,"children":29847},{"className":29846},[844],[29848],{"type":155,"value":1045},{"type":150,"tag":169,"props":29850,"children":29852},{"className":29851,"style":839},[558],[],{"type":150,"tag":169,"props":29854,"children":29856},{"className":29855},[237],[29857,29861,29866,29871,29880,29884,29889],{"type":150,"tag":169,"props":29858,"children":29860},{"className":29859,"style":5854},[242],[],{"type":150,"tag":169,"props":29862,"children":29864},{"className":29863},[248],[29865],{"type":155,"value":24940},{"type":150,"tag":169,"props":29867,"children":29869},{"className":29868},[558],[29870],{"type":155,"value":8763},{"type":150,"tag":169,"props":29872,"children":29874},{"className":29873},[248,155],[29875],{"type":150,"tag":169,"props":29876,"children":29878},{"className":29877},[248],[29879],{"type":155,"value":29613},{"type":150,"tag":169,"props":29881,"children":29883},{"className":29882,"style":839},[558],[],{"type":150,"tag":169,"props":29885,"children":29887},{"className":29886},[844],[29888],{"type":155,"value":29472},{"type":150,"tag":169,"props":29890,"children":29892},{"className":29891,"style":839},[558],[],{"type":150,"tag":169,"props":29894,"children":29896},{"className":29895},[237],[29897,29901,29906,29910,29915],{"type":150,"tag":169,"props":29898,"children":29900},{"className":29899,"style":5854},[242],[],{"type":150,"tag":169,"props":29902,"children":29904},{"className":29903,"style":5780},[248,489],[29905],{"type":155,"value":5591},{"type":150,"tag":169,"props":29907,"children":29909},{"className":29908,"style":839},[558],[],{"type":150,"tag":169,"props":29911,"children":29913},{"className":29912},[844],[29914],{"type":155,"value":6050},{"type":150,"tag":169,"props":29916,"children":29918},{"className":29917,"style":839},[558],[],{"type":150,"tag":169,"props":29920,"children":29922},{"className":29921},[237],[29923,29927,29932,29937],{"type":150,"tag":169,"props":29924,"children":29926},{"className":29925,"style":5854},[242],[],{"type":150,"tag":169,"props":29928,"children":29930},{"className":29929},[248],[29931],{"type":155,"value":29753},{"type":150,"tag":169,"props":29933,"children":29935},{"className":29934},[558],[29936],{"type":155,"value":8763},{"type":150,"tag":169,"props":29938,"children":29940},{"className":29939},[248,155],[29941],{"type":150,"tag":169,"props":29942,"children":29944},{"className":29943},[248,297],[29945],{"type":155,"value":29680},{"type":150,"tag":151,"props":29947,"children":29948},{},[29949],{"type":155,"value":29950},"既然两边都是 memory bound，加速比就等于字节数之比：",{"type":150,"tag":22368,"props":29952,"children":29953},{},[29954,30010],{"type":150,"tag":22372,"props":29955,"children":29956},{},[29957],{"type":150,"tag":22376,"props":29958,"children":29959},{},[29960,30005],{"type":150,"tag":22380,"props":29961,"children":29962},{},[29963],{"type":150,"tag":169,"props":29964,"children":29966},{"className":29965},[177],[29967,29988],{"type":150,"tag":169,"props":29968,"children":29970},{"className":29969},[182],[29971],{"type":150,"tag":185,"props":29972,"children":29973},{"xmlns":187},[29974],{"type":150,"tag":191,"props":29975,"children":29976},{},[29977,29984],{"type":150,"tag":195,"props":29978,"children":29979},{},[29980],{"type":150,"tag":405,"props":29981,"children":29982},{},[29983],{"type":155,"value":5591},{"type":150,"tag":223,"props":29985,"children":29986},{"encoding":225},[29987],{"type":155,"value":5591},{"type":150,"tag":169,"props":29989,"children":29991},{"className":29990,"ariaHidden":214},[232],[29992],{"type":150,"tag":169,"props":29993,"children":29995},{"className":29994},[237],[29996,30000],{"type":150,"tag":169,"props":29997,"children":29999},{"className":29998,"style":5854},[242],[],{"type":150,"tag":169,"props":30001,"children":30003},{"className":30002,"style":5780},[248,489],[30004],{"type":155,"value":5591},{"type":150,"tag":22380,"props":30006,"children":30007},{},[30008],{"type":155,"value":30009},"MLA 相对 MHA 的每 token 加速比",{"type":150,"tag":22394,"props":30011,"children":30012},{},[30013,30025,30037,30049],{"type":150,"tag":22376,"props":30014,"children":30015},{},[30016,30020],{"type":150,"tag":22401,"props":30017,"children":30018},{},[30019],{"type":155,"value":27081},{"type":150,"tag":22401,"props":30021,"children":30022},{},[30023],{"type":155,"value":30024},"1.4×",{"type":150,"tag":22376,"props":30026,"children":30027},{},[30028,30032],{"type":150,"tag":22401,"props":30029,"children":30030},{},[30031],{"type":155,"value":27109},{"type":150,"tag":22401,"props":30033,"children":30034},{},[30035],{"type":155,"value":30036},"3.9×",{"type":150,"tag":22376,"props":30038,"children":30039},{},[30040,30044],{"type":150,"tag":22401,"props":30041,"children":30042},{},[30043],{"type":155,"value":27136},{"type":150,"tag":22401,"props":30045,"children":30046},{},[30047],{"type":155,"value":30048},"10.9×",{"type":150,"tag":22376,"props":30050,"children":30051},{},[30052,30056],{"type":150,"tag":22401,"props":30053,"children":30054},{},[30055],{"type":155,"value":27163},{"type":150,"tag":22401,"props":30057,"children":30058},{},[30059],{"type":155,"value":30060},"36.4×",{"type":150,"tag":151,"props":30062,"children":30063},{},[30064,30066],{"type":155,"value":30065},"所以答案不是\"足够大时才划算\"，而是一条具体的曲线：",{"type":150,"tag":158,"props":30067,"children":30068},{},[30069],{"type":155,"value":30070},"4k 时几乎白干（1.4×），32k 开始有意义（3.9×），128k 是 11 倍，1M 是 36 倍。",{"type":150,"tag":151,"props":30072,"children":30073},{},[30074],{"type":155,"value":30075},"这条曲线顺带解释了 MLA 的时间线。它 2024 年出来的时候，主流 context 还在 4k 到 32k 之间，那一段的收益并不惊人。真正让它值回票价的是后面 128k、1M 的军备竞赛。",{"type":150,"tag":151,"props":30077,"children":30078},{},[30079,30081,30251,30253,30502],{"type":155,"value":30080},"另外还有一件事能说明这个设计有多自觉。一般 LLM 架构满足 ",{"type":150,"tag":169,"props":30082,"children":30084},{"className":30083},[177],[30085,30130],{"type":150,"tag":169,"props":30086,"children":30088},{"className":30087},[182],[30089],{"type":150,"tag":185,"props":30090,"children":30091},{"xmlns":187},[30092],{"type":150,"tag":191,"props":30093,"children":30094},{},[30095,30125],{"type":150,"tag":195,"props":30096,"children":30097},{},[30098,30102,30106,30117,30121],{"type":150,"tag":405,"props":30099,"children":30100},{},[30101],{"type":155,"value":926},{"type":150,"tag":417,"props":30103,"children":30104},{},[30105],{"type":155,"value":5554},{"type":150,"tag":401,"props":30107,"children":30108},{},[30109,30113],{"type":150,"tag":405,"props":30110,"children":30111},{},[30112],{"type":155,"value":759},{"type":150,"tag":405,"props":30114,"children":30115},{},[30116],{"type":155,"value":1876},{"type":150,"tag":417,"props":30118,"children":30119},{},[30120],{"type":155,"value":1045},{"type":150,"tag":405,"props":30122,"children":30123},{},[30124],{"type":155,"value":759},{"type":150,"tag":223,"props":30126,"children":30127},{"encoding":225},[30128],{"type":155,"value":30129},"h \\times d_k = d",{"type":150,"tag":169,"props":30131,"children":30133},{"className":30132,"ariaHidden":214},[232],[30134,30160,30238],{"type":150,"tag":169,"props":30135,"children":30137},{"className":30136},[237],[30138,30142,30147,30151,30156],{"type":150,"tag":169,"props":30139,"children":30141},{"className":30140,"style":5671},[242],[],{"type":150,"tag":169,"props":30143,"children":30145},{"className":30144},[248,489],[30146],{"type":155,"value":926},{"type":150,"tag":169,"props":30148,"children":30150},{"className":30149,"style":5167},[558],[],{"type":150,"tag":169,"props":30152,"children":30154},{"className":30153},[5172],[30155],{"type":155,"value":5554},{"type":150,"tag":169,"props":30157,"children":30159},{"className":30158,"style":5167},[558],[],{"type":150,"tag":169,"props":30161,"children":30163},{"className":30162},[237],[30164,30168,30225,30229,30234],{"type":150,"tag":169,"props":30165,"children":30167},{"className":30166,"style":4935},[242],[],{"type":150,"tag":169,"props":30169,"children":30171},{"className":30170},[248],[30172,30177],{"type":150,"tag":169,"props":30173,"children":30175},{"className":30174},[248,489],[30176],{"type":155,"value":759},{"type":150,"tag":169,"props":30178,"children":30180},{"className":30179},[495],[30181],{"type":150,"tag":169,"props":30182,"children":30184},{"className":30183},[253,254],[30185,30214],{"type":150,"tag":169,"props":30186,"children":30188},{"className":30187},[259],[30189,30209],{"type":150,"tag":169,"props":30190,"children":30192},{"className":30191,"style":674},[264],[30193],{"type":150,"tag":169,"props":30194,"children":30195},{"style":512},[30196,30200],{"type":150,"tag":169,"props":30197,"children":30199},{"className":30198,"style":517},[274],[],{"type":150,"tag":169,"props":30201,"children":30203},{"className":30202},[522,523,524,525],[30204],{"type":150,"tag":169,"props":30205,"children":30207},{"className":30206,"style":2427},[248,489,525],[30208],{"type":155,"value":1876},{"type":150,"tag":169,"props":30210,"children":30212},{"className":30211},[355],[30213],{"type":155,"value":358},{"type":150,"tag":169,"props":30215,"children":30217},{"className":30216},[259],[30218],{"type":150,"tag":169,"props":30219,"children":30221},{"className":30220,"style":544},[264],[30222],{"type":150,"tag":169,"props":30223,"children":30224},{},[],{"type":150,"tag":169,"props":30226,"children":30228},{"className":30227,"style":839},[558],[],{"type":150,"tag":169,"props":30230,"children":30232},{"className":30231},[844],[30233],{"type":155,"value":1045},{"type":150,"tag":169,"props":30235,"children":30237},{"className":30236,"style":839},[558],[],{"type":150,"tag":169,"props":30239,"children":30241},{"className":30240},[237],[30242,30246],{"type":150,"tag":169,"props":30243,"children":30245},{"className":30244,"style":942},[242],[],{"type":150,"tag":169,"props":30247,"children":30249},{"className":30248},[248,489],[30250],{"type":155,"value":759},{"type":155,"value":30252},"，但 DeepSeek-V2 是 ",{"type":150,"tag":169,"props":30254,"children":30256},{"className":30255},[177],[30257,30327],{"type":150,"tag":169,"props":30258,"children":30260},{"className":30259},[182],[30261],{"type":150,"tag":185,"props":30262,"children":30263},{"xmlns":187},[30264],{"type":150,"tag":191,"props":30265,"children":30266},{},[30267,30322],{"type":150,"tag":195,"props":30268,"children":30269},{},[30270,30281,30285,30289,30293,30297,30301,30306,30310,30314,30318],{"type":150,"tag":401,"props":30271,"children":30272},{},[30273,30277],{"type":150,"tag":405,"props":30274,"children":30275},{},[30276],{"type":155,"value":759},{"type":150,"tag":405,"props":30278,"children":30279},{},[30280],{"type":155,"value":1876},{"type":150,"tag":417,"props":30282,"children":30283},{},[30284],{"type":155,"value":1045},{"type":150,"tag":411,"props":30286,"children":30287},{},[30288],{"type":155,"value":6045},{"type":150,"tag":417,"props":30290,"children":30291},{"separator":214},[30292],{"type":155,"value":421},{"type":150,"tag":405,"props":30294,"children":30295},{},[30296],{"type":155,"value":759},{"type":150,"tag":417,"props":30298,"children":30299},{},[30300],{"type":155,"value":1045},{"type":150,"tag":411,"props":30302,"children":30303},{},[30304],{"type":155,"value":30305},"5120",{"type":150,"tag":417,"props":30307,"children":30308},{"separator":214},[30309],{"type":155,"value":421},{"type":150,"tag":405,"props":30311,"children":30312},{},[30313],{"type":155,"value":926},{"type":150,"tag":417,"props":30315,"children":30316},{},[30317],{"type":155,"value":1045},{"type":150,"tag":411,"props":30319,"children":30320},{},[30321],{"type":155,"value":6045},{"type":150,"tag":223,"props":30323,"children":30324},{"encoding":225},[30325],{"type":155,"value":30326},"d_k = 128, d = 5120, h = 128",{"type":150,"tag":169,"props":30328,"children":30330},{"className":30329,"ariaHidden":214},[232],[30331,30409,30449,30489],{"type":150,"tag":169,"props":30332,"children":30334},{"className":30333},[237],[30335,30339,30396,30400,30405],{"type":150,"tag":169,"props":30336,"children":30338},{"className":30337,"style":4935},[242],[],{"type":150,"tag":169,"props":30340,"children":30342},{"className":30341},[248],[30343,30348],{"type":150,"tag":169,"props":30344,"children":30346},{"className":30345},[248,489],[30347],{"type":155,"value":759},{"type":150,"tag":169,"props":30349,"children":30351},{"className":30350},[495],[30352],{"type":150,"tag":169,"props":30353,"children":30355},{"className":30354},[253,254],[30356,30385],{"type":150,"tag":169,"props":30357,"children":30359},{"className":30358},[259],[30360,30380],{"type":150,"tag":169,"props":30361,"children":30363},{"className":30362,"style":674},[264],[30364],{"type":150,"tag":169,"props":30365,"children":30366},{"style":512},[30367,30371],{"type":150,"tag":169,"props":30368,"children":30370},{"className":30369,"style":517},[274],[],{"type":150,"tag":169,"props":30372,"children":30374},{"className":30373},[522,523,524,525],[30375],{"type":150,"tag":169,"props":30376,"children":30378},{"className":30377,"style":2427},[248,489,525],[30379],{"type":155,"value":1876},{"type":150,"tag":169,"props":30381,"children":30383},{"className":30382},[355],[30384],{"type":155,"value":358},{"type":150,"tag":169,"props":30386,"children":30388},{"className":30387},[259],[30389],{"type":150,"tag":169,"props":30390,"children":30392},{"className":30391,"style":544},[264],[30393],{"type":150,"tag":169,"props":30394,"children":30395},{},[],{"type":150,"tag":169,"props":30397,"children":30399},{"className":30398,"style":839},[558],[],{"type":150,"tag":169,"props":30401,"children":30403},{"className":30402},[844],[30404],{"type":155,"value":1045},{"type":150,"tag":169,"props":30406,"children":30408},{"className":30407,"style":839},[558],[],{"type":150,"tag":169,"props":30410,"children":30412},{"className":30411},[237],[30413,30417,30422,30427,30431,30436,30440,30445],{"type":150,"tag":169,"props":30414,"children":30416},{"className":30415,"style":1121},[242],[],{"type":150,"tag":169,"props":30418,"children":30420},{"className":30419},[248],[30421],{"type":155,"value":6045},{"type":150,"tag":169,"props":30423,"children":30425},{"className":30424},[552],[30426],{"type":155,"value":421},{"type":150,"tag":169,"props":30428,"children":30430},{"className":30429,"style":559},[558],[],{"type":150,"tag":169,"props":30432,"children":30434},{"className":30433},[248,489],[30435],{"type":155,"value":759},{"type":150,"tag":169,"props":30437,"children":30439},{"className":30438,"style":839},[558],[],{"type":150,"tag":169,"props":30441,"children":30443},{"className":30442},[844],[30444],{"type":155,"value":1045},{"type":150,"tag":169,"props":30446,"children":30448},{"className":30447,"style":839},[558],[],{"type":150,"tag":169,"props":30450,"children":30452},{"className":30451},[237],[30453,30457,30462,30467,30471,30476,30480,30485],{"type":150,"tag":169,"props":30454,"children":30456},{"className":30455,"style":1121},[242],[],{"type":150,"tag":169,"props":30458,"children":30460},{"className":30459},[248],[30461],{"type":155,"value":30305},{"type":150,"tag":169,"props":30463,"children":30465},{"className":30464},[552],[30466],{"type":155,"value":421},{"type":150,"tag":169,"props":30468,"children":30470},{"className":30469,"style":559},[558],[],{"type":150,"tag":169,"props":30472,"children":30474},{"className":30473},[248,489],[30475],{"type":155,"value":926},{"type":150,"tag":169,"props":30477,"children":30479},{"className":30478,"style":839},[558],[],{"type":150,"tag":169,"props":30481,"children":30483},{"className":30482},[844],[30484],{"type":155,"value":1045},{"type":150,"tag":169,"props":30486,"children":30488},{"className":30487,"style":839},[558],[],{"type":150,"tag":169,"props":30490,"children":30492},{"className":30491},[237],[30493,30497],{"type":150,"tag":169,"props":30494,"children":30496},{"className":30495,"style":5186},[242],[],{"type":150,"tag":169,"props":30498,"children":30500},{"className":30499},[248],[30501],{"type":155,"value":6045},{"type":155,"value":30503},"，head 数是一般设置的 3 倍。",{"type":150,"tag":151,"props":30505,"children":30506},{},[30507,30509,30551,30553,30595],{"type":155,"value":30508},"因为 MLA 的 KV Cache 大小跟 ",{"type":150,"tag":169,"props":30510,"children":30512},{"className":30511},[177],[30513,30534],{"type":150,"tag":169,"props":30514,"children":30516},{"className":30515},[182],[30517],{"type":150,"tag":185,"props":30518,"children":30519},{"xmlns":187},[30520],{"type":150,"tag":191,"props":30521,"children":30522},{},[30523,30530],{"type":150,"tag":195,"props":30524,"children":30525},{},[30526],{"type":150,"tag":405,"props":30527,"children":30528},{},[30529],{"type":155,"value":926},{"type":150,"tag":223,"props":30531,"children":30532},{"encoding":225},[30533],{"type":155,"value":926},{"type":150,"tag":169,"props":30535,"children":30537},{"className":30536,"ariaHidden":214},[232],[30538],{"type":150,"tag":169,"props":30539,"children":30541},{"className":30540},[237],[30542,30546],{"type":150,"tag":169,"props":30543,"children":30545},{"className":30544,"style":942},[242],[],{"type":150,"tag":169,"props":30547,"children":30549},{"className":30548},[248,489],[30550],{"type":155,"value":926},{"type":155,"value":30552}," 无关。增大 ",{"type":150,"tag":169,"props":30554,"children":30556},{"className":30555},[177],[30557,30578],{"type":150,"tag":169,"props":30558,"children":30560},{"className":30559},[182],[30561],{"type":150,"tag":185,"props":30562,"children":30563},{"xmlns":187},[30564],{"type":150,"tag":191,"props":30565,"children":30566},{},[30567,30574],{"type":150,"tag":195,"props":30568,"children":30569},{},[30570],{"type":150,"tag":405,"props":30571,"children":30572},{},[30573],{"type":155,"value":926},{"type":150,"tag":223,"props":30575,"children":30576},{"encoding":225},[30577],{"type":155,"value":926},{"type":150,"tag":169,"props":30579,"children":30581},{"className":30580,"ariaHidden":214},[232],[30582],{"type":150,"tag":169,"props":30583,"children":30585},{"className":30584},[237],[30586,30590],{"type":150,"tag":169,"props":30587,"children":30589},{"className":30588,"style":942},[242],[],{"type":150,"tag":169,"props":30591,"children":30593},{"className":30592},[248,489],[30594],{"type":155,"value":926},{"type":155,"value":30596}," 只增加计算量和提升模型能力，不增加 KV Cache，所以不会带来速度瓶颈。MLA 完成的思想升级可以写成一句话：",{"type":150,"tag":169,"props":30598,"children":30600},{"className":30599},[172],[30601],{"type":150,"tag":169,"props":30602,"children":30604},{"className":30603},[177],[30605,30640],{"type":150,"tag":169,"props":30606,"children":30608},{"className":30607},[182],[30609],{"type":150,"tag":185,"props":30610,"children":30611},{"xmlns":187,"display":188},[30612],{"type":150,"tag":191,"props":30613,"children":30614},{},[30615,30635],{"type":150,"tag":195,"props":30616,"children":30617},{},[30618],{"type":150,"tag":199,"props":30619,"children":30620},{"notation":201},[30621],{"type":150,"tag":204,"props":30622,"children":30623},{"scriptlevel":206,"displaystyle":207},[30624],{"type":150,"tag":204,"props":30625,"children":30626},{"scriptlevel":206,"displaystyle":207},[30627],{"type":150,"tag":204,"props":30628,"children":30629},{"scriptlevel":206,"displaystyle":214},[30630],{"type":150,"tag":217,"props":30631,"children":30632},{},[30633],{"type":155,"value":30634},"从离散的 head sharing，到连续的 representation compression",{"type":150,"tag":223,"props":30636,"children":30637},{"encoding":225},[30638],{"type":155,"value":30639},"\\boxed{\\text{从离散的 head sharing，到连续的 representation compression}}",{"type":150,"tag":169,"props":30641,"children":30643},{"className":30642,"ariaHidden":214},[232],[30644],{"type":150,"tag":169,"props":30645,"children":30647},{"className":30646},[237],[30648,30653],{"type":150,"tag":169,"props":30649,"children":30652},{"className":30650,"style":30651},[242],"height:1.5689em;vertical-align:-0.5344em;",[],{"type":150,"tag":169,"props":30654,"children":30656},{"className":30655},[248],[30657],{"type":150,"tag":169,"props":30658,"children":30660},{"className":30659},[253,254],[30661,30737],{"type":150,"tag":169,"props":30662,"children":30664},{"className":30663},[259],[30665,30732],{"type":150,"tag":169,"props":30666,"children":30669},{"className":30667,"style":30668},[264],"height:1.0344em;",[30670,30719],{"type":150,"tag":169,"props":30671,"children":30673},{"style":30672},"top:-3.5689em;",[30674,30679],{"type":150,"tag":169,"props":30675,"children":30678},{"className":30676,"style":30677},[274],"height:3.5689em;",[],{"type":150,"tag":169,"props":30680,"children":30682},{"className":30681},[280],[30683],{"type":150,"tag":169,"props":30684,"children":30686},{"className":30685},[248],[30687],{"type":150,"tag":169,"props":30688,"children":30690},{"className":30689},[248],[30691],{"type":150,"tag":169,"props":30692,"children":30694},{"className":30693},[248,155],[30695,30701,30707,30713],{"type":150,"tag":169,"props":30696,"children":30698},{"className":30697},[248,297],[30699],{"type":155,"value":30700},"从离散的",{"type":150,"tag":169,"props":30702,"children":30704},{"className":30703},[248],[30705],{"type":155,"value":30706}," head sharing",{"type":150,"tag":169,"props":30708,"children":30710},{"className":30709},[248,297],[30711],{"type":155,"value":30712},"，到连续的",{"type":150,"tag":169,"props":30714,"children":30716},{"className":30715},[248],[30717],{"type":155,"value":30718}," representation compression",{"type":150,"tag":169,"props":30720,"children":30722},{"style":30721},"top:-3.0344em;",[30723,30727],{"type":150,"tag":169,"props":30724,"children":30726},{"className":30725,"style":30677},[274],[],{"type":150,"tag":169,"props":30728,"children":30731},{"className":30729,"style":30730},[348,349],"height:1.5689em;border-style:solid;border-width:0.04em;",[],{"type":150,"tag":169,"props":30733,"children":30735},{"className":30734},[355],[30736],{"type":155,"value":358},{"type":150,"tag":169,"props":30738,"children":30740},{"className":30739},[259],[30741],{"type":150,"tag":169,"props":30742,"children":30744},{"className":30743,"style":366},[264],[30745],{"type":150,"tag":169,"props":30746,"children":30747},{},[],{"type":150,"tag":372,"props":30749,"children":30751},{"id":30750},"csa-与-hca",[30752],{"type":155,"value":30753},"CSA 与 HCA",{"type":150,"tag":30755,"props":30756,"children":30758},"h3",{"id":30757},"压-sequence-轴",[30759],{"type":155,"value":30760},"压 Sequence 轴",{"type":150,"tag":151,"props":30762,"children":30763},{},[30764,30766,31065,31067,31109,31111,31188],{"type":155,"value":30765},"MLA 压小的是每个 token 存的东西。但 ",{"type":150,"tag":169,"props":30767,"children":30769},{"className":30768},[177],[30770,30841],{"type":150,"tag":169,"props":30771,"children":30773},{"className":30772},[182],[30774],{"type":150,"tag":185,"props":30775,"children":30776},{"xmlns":187},[30777],{"type":150,"tag":191,"props":30778,"children":30779},{},[30780,30836],{"type":150,"tag":195,"props":30781,"children":30782},{},[30783,30794,30798,30809,30813,30817,30821,30825],{"type":150,"tag":401,"props":30784,"children":30785},{},[30786,30790],{"type":150,"tag":405,"props":30787,"children":30788},{},[30789],{"type":155,"value":10046},{"type":150,"tag":411,"props":30791,"children":30792},{},[30793],{"type":155,"value":415},{"type":150,"tag":417,"props":30795,"children":30796},{"separator":214},[30797],{"type":155,"value":421},{"type":150,"tag":401,"props":30799,"children":30800},{},[30801,30805],{"type":150,"tag":405,"props":30802,"children":30803},{},[30804],{"type":155,"value":10046},{"type":150,"tag":411,"props":30806,"children":30807},{},[30808],{"type":155,"value":433},{"type":150,"tag":417,"props":30810,"children":30811},{"separator":214},[30812],{"type":155,"value":421},{"type":150,"tag":417,"props":30814,"children":30815},{},[30816],{"type":155,"value":442},{"type":150,"tag":217,"props":30818,"children":30819},{},[30820],{"type":155,"value":447},{"type":150,"tag":417,"props":30822,"children":30823},{"separator":214},[30824],{"type":155,"value":421},{"type":150,"tag":401,"props":30826,"children":30827},{},[30828,30832],{"type":150,"tag":405,"props":30829,"children":30830},{},[30831],{"type":155,"value":10046},{"type":150,"tag":405,"props":30833,"children":30834},{},[30835],{"type":155,"value":5591},{"type":150,"tag":223,"props":30837,"children":30838},{"encoding":225},[30839],{"type":155,"value":30840},"c_1, c_2, \\cdots, c_N",{"type":150,"tag":169,"props":30842,"children":30844},{"className":30843,"ariaHidden":214},[232],[30845],{"type":150,"tag":169,"props":30846,"children":30848},{"className":30847},[237],[30849,30853,30910,30915,30919,30976,30981,30985,30990,30994,30998,31003,31007],{"type":150,"tag":169,"props":30850,"children":30852},{"className":30851,"style":480},[242],[],{"type":150,"tag":169,"props":30854,"children":30856},{"className":30855},[248],[30857,30862],{"type":150,"tag":169,"props":30858,"children":30860},{"className":30859},[248,489],[30861],{"type":155,"value":10046},{"type":150,"tag":169,"props":30863,"children":30865},{"className":30864},[495],[30866],{"type":150,"tag":169,"props":30867,"children":30869},{"className":30868},[253,254],[30870,30899],{"type":150,"tag":169,"props":30871,"children":30873},{"className":30872},[259],[30874,30894],{"type":150,"tag":169,"props":30875,"children":30877},{"className":30876,"style":508},[264],[30878],{"type":150,"tag":169,"props":30879,"children":30880},{"style":512},[30881,30885],{"type":150,"tag":169,"props":30882,"children":30884},{"className":30883,"style":517},[274],[],{"type":150,"tag":169,"props":30886,"children":30888},{"className":30887},[522,523,524,525],[30889],{"type":150,"tag":169,"props":30890,"children":30892},{"className":30891},[248,525],[30893],{"type":155,"value":415},{"type":150,"tag":169,"props":30895,"children":30897},{"className":30896},[355],[30898],{"type":155,"value":358},{"type":150,"tag":169,"props":30900,"children":30902},{"className":30901},[259],[30903],{"type":150,"tag":169,"props":30904,"children":30906},{"className":30905,"style":544},[264],[30907],{"type":150,"tag":169,"props":30908,"children":30909},{},[],{"type":150,"tag":169,"props":30911,"children":30913},{"className":30912},[552],[30914],{"type":155,"value":421},{"type":150,"tag":169,"props":30916,"children":30918},{"className":30917,"style":559},[558],[],{"type":150,"tag":169,"props":30920,"children":30922},{"className":30921},[248],[30923,30928],{"type":150,"tag":169,"props":30924,"children":30926},{"className":30925},[248,489],[30927],{"type":155,"value":10046},{"type":150,"tag":169,"props":30929,"children":30931},{"className":30930},[495],[30932],{"type":150,"tag":169,"props":30933,"children":30935},{"className":30934},[253,254],[30936,30965],{"type":150,"tag":169,"props":30937,"children":30939},{"className":30938},[259],[30940,30960],{"type":150,"tag":169,"props":30941,"children":30943},{"className":30942,"style":508},[264],[30944],{"type":150,"tag":169,"props":30945,"children":30946},{"style":512},[30947,30951],{"type":150,"tag":169,"props":30948,"children":30950},{"className":30949,"style":517},[274],[],{"type":150,"tag":169,"props":30952,"children":30954},{"className":30953},[522,523,524,525],[30955],{"type":150,"tag":169,"props":30956,"children":30958},{"className":30957},[248,525],[30959],{"type":155,"value":433},{"type":150,"tag":169,"props":30961,"children":30963},{"className":30962},[355],[30964],{"type":155,"value":358},{"type":150,"tag":169,"props":30966,"children":30968},{"className":30967},[259],[30969],{"type":150,"tag":169,"props":30970,"children":30972},{"className":30971,"style":544},[264],[30973],{"type":150,"tag":169,"props":30974,"children":30975},{},[],{"type":150,"tag":169,"props":30977,"children":30979},{"className":30978},[552],[30980],{"type":155,"value":421},{"type":150,"tag":169,"props":30982,"children":30984},{"className":30983,"style":559},[558],[],{"type":150,"tag":169,"props":30986,"children":30988},{"className":30987},[630],[30989],{"type":155,"value":442},{"type":150,"tag":169,"props":30991,"children":30993},{"className":30992,"style":559},[558],[],{"type":150,"tag":169,"props":30995,"children":30997},{"className":30996,"style":559},[558],[],{"type":150,"tag":169,"props":30999,"children":31001},{"className":31000},[552],[31002],{"type":155,"value":421},{"type":150,"tag":169,"props":31004,"children":31006},{"className":31005,"style":559},[558],[],{"type":150,"tag":169,"props":31008,"children":31010},{"className":31009},[248],[31011,31016],{"type":150,"tag":169,"props":31012,"children":31014},{"className":31013},[248,489],[31015],{"type":155,"value":10046},{"type":150,"tag":169,"props":31017,"children":31019},{"className":31018},[495],[31020],{"type":150,"tag":169,"props":31021,"children":31023},{"className":31022},[253,254],[31024,31054],{"type":150,"tag":169,"props":31025,"children":31027},{"className":31026},[259],[31028,31049],{"type":150,"tag":169,"props":31029,"children":31032},{"className":31030,"style":31031},[264],"height:0.3283em;",[31033],{"type":150,"tag":169,"props":31034,"children":31035},{"style":512},[31036,31040],{"type":150,"tag":169,"props":31037,"children":31039},{"className":31038,"style":517},[274],[],{"type":150,"tag":169,"props":31041,"children":31043},{"className":31042},[522,523,524,525],[31044],{"type":150,"tag":169,"props":31045,"children":31047},{"className":31046,"style":5780},[248,489,525],[31048],{"type":155,"value":5591},{"type":150,"tag":169,"props":31050,"children":31052},{"className":31051},[355],[31053],{"type":155,"value":358},{"type":150,"tag":169,"props":31055,"children":31057},{"className":31056},[259],[31058],{"type":150,"tag":169,"props":31059,"children":31061},{"className":31060,"style":544},[264],[31062],{"type":150,"tag":169,"props":31063,"children":31064},{},[],{"type":155,"value":31066}," 仍然有 ",{"type":150,"tag":169,"props":31068,"children":31070},{"className":31069},[177],[31071,31092],{"type":150,"tag":169,"props":31072,"children":31074},{"className":31073},[182],[31075],{"type":150,"tag":185,"props":31076,"children":31077},{"xmlns":187},[31078],{"type":150,"tag":191,"props":31079,"children":31080},{},[31081,31088],{"type":150,"tag":195,"props":31082,"children":31083},{},[31084],{"type":150,"tag":405,"props":31085,"children":31086},{},[31087],{"type":155,"value":5591},{"type":150,"tag":223,"props":31089,"children":31090},{"encoding":225},[31091],{"type":155,"value":5591},{"type":150,"tag":169,"props":31093,"children":31095},{"className":31094,"ariaHidden":214},[232],[31096],{"type":150,"tag":169,"props":31097,"children":31099},{"className":31098},[237],[31100,31104],{"type":150,"tag":169,"props":31101,"children":31103},{"className":31102,"style":5854},[242],[],{"type":150,"tag":169,"props":31105,"children":31107},{"className":31106,"style":5780},[248,489],[31108],{"type":155,"value":5591},{"type":155,"value":31110}," 个。如果每个 query 还是要看全部历史，那 ",{"type":150,"tag":169,"props":31112,"children":31114},{"className":31113},[177],[31115,31145],{"type":150,"tag":169,"props":31116,"children":31118},{"className":31117},[182],[31119],{"type":150,"tag":185,"props":31120,"children":31121},{"xmlns":187},[31122],{"type":150,"tag":191,"props":31123,"children":31124},{},[31125,31140],{"type":150,"tag":195,"props":31126,"children":31127},{},[31128,31132,31136],{"type":150,"tag":405,"props":31129,"children":31130},{},[31131],{"type":155,"value":5591},{"type":150,"tag":417,"props":31133,"children":31134},{},[31135],{"type":155,"value":5554},{"type":150,"tag":405,"props":31137,"children":31138},{},[31139],{"type":155,"value":5591},{"type":150,"tag":223,"props":31141,"children":31142},{"encoding":225},[31143],{"type":155,"value":31144},"N \\times N",{"type":150,"tag":169,"props":31146,"children":31148},{"className":31147,"ariaHidden":214},[232],[31149,31175],{"type":150,"tag":169,"props":31150,"children":31152},{"className":31151},[237],[31153,31157,31162,31166,31171],{"type":150,"tag":169,"props":31154,"children":31156},{"className":31155,"style":5644},[242],[],{"type":150,"tag":169,"props":31158,"children":31160},{"className":31159,"style":5780},[248,489],[31161],{"type":155,"value":5591},{"type":150,"tag":169,"props":31163,"children":31165},{"className":31164,"style":5167},[558],[],{"type":150,"tag":169,"props":31167,"children":31169},{"className":31168},[5172],[31170],{"type":155,"value":5554},{"type":150,"tag":169,"props":31172,"children":31174},{"className":31173,"style":5167},[558],[],{"type":150,"tag":169,"props":31176,"children":31178},{"className":31177},[237],[31179,31183],{"type":150,"tag":169,"props":31180,"children":31182},{"className":31181,"style":5854},[242],[],{"type":150,"tag":169,"props":31184,"children":31186},{"className":31185,"style":5780},[248,489],[31187],{"type":155,"value":5591},{"type":155,"value":31189}," 还在。",{"type":150,"tag":151,"props":31191,"children":31192},{},[31193],{"type":150,"tag":158,"props":31194,"children":31195},{},[31196],{"type":155,"value":31197},"MLA 解决的是 KV 的宽度，没解决历史的长度。",{"type":150,"tag":151,"props":31199,"children":31200},{},[31201],{"type":155,"value":31202},"把 KV Cache 想成一个二维矩阵：",{"type":150,"tag":169,"props":31204,"children":31206},{"className":31205},[172],[31207],{"type":150,"tag":169,"props":31208,"children":31210},{"className":31209},[177],[31211,31243],{"type":150,"tag":169,"props":31212,"children":31214},{"className":31213},[182],[31215],{"type":150,"tag":185,"props":31216,"children":31217},{"xmlns":187,"display":188},[31218],{"type":150,"tag":191,"props":31219,"children":31220},{},[31221,31238],{"type":150,"tag":195,"props":31222,"children":31223},{},[31224,31229,31233],{"type":150,"tag":217,"props":31225,"children":31226},{},[31227],{"type":155,"value":31228},"Sequence",{"type":150,"tag":417,"props":31230,"children":31231},{},[31232],{"type":155,"value":5554},{"type":150,"tag":217,"props":31234,"children":31235},{},[31236],{"type":155,"value":31237},"Feature",{"type":150,"tag":223,"props":31239,"children":31240},{"encoding":225},[31241],{"type":155,"value":31242},"\\text{Sequence} \\times \\text{Feature}",{"type":150,"tag":169,"props":31244,"children":31246},{"className":31245,"ariaHidden":214},[232],[31247,31277],{"type":150,"tag":169,"props":31248,"children":31250},{"className":31249},[237],[31251,31255,31264,31268,31273],{"type":150,"tag":169,"props":31252,"children":31254},{"className":31253,"style":11723},[242],[],{"type":150,"tag":169,"props":31256,"children":31258},{"className":31257},[248,155],[31259],{"type":150,"tag":169,"props":31260,"children":31262},{"className":31261},[248],[31263],{"type":155,"value":31228},{"type":150,"tag":169,"props":31265,"children":31267},{"className":31266,"style":5167},[558],[],{"type":150,"tag":169,"props":31269,"children":31271},{"className":31270},[5172],[31272],{"type":155,"value":5554},{"type":150,"tag":169,"props":31274,"children":31276},{"className":31275,"style":5167},[558],[],{"type":150,"tag":169,"props":31278,"children":31280},{"className":31279},[237],[31281,31285],{"type":150,"tag":169,"props":31282,"children":31284},{"className":31283,"style":5854},[242],[],{"type":150,"tag":169,"props":31286,"children":31288},{"className":31287},[248,155],[31289],{"type":150,"tag":169,"props":31290,"children":31292},{"className":31291},[248],[31293],{"type":155,"value":31237},{"type":150,"tag":151,"props":31295,"children":31296},{},[31297],{"type":155,"value":31298},"MQA、GQA、MLA 压的都是 Feature 轴。V4 开始压 Sequence 轴。V4 改成了三类 attention：",{"type":150,"tag":23833,"props":31300,"children":31301},{},[31302,31312,31322],{"type":150,"tag":23837,"props":31303,"children":31304},{},[31305,31310],{"type":150,"tag":158,"props":31306,"children":31307},{},[31308],{"type":155,"value":31309},"CSA",{"type":155,"value":31311},"：先按大约 4:1 沿序列压缩，再用 Lightning Indexer 给每个 query 挑少量 top-k compressed blocks",{"type":150,"tag":23837,"props":31313,"children":31314},{},[31315,31320],{"type":150,"tag":158,"props":31316,"children":31317},{},[31318],{"type":155,"value":31319},"HCA",{"type":155,"value":31321},"：用更激进的序列压缩（约 128 倍），让 query 对压缩后的全局历史做 attention",{"type":150,"tag":23837,"props":31323,"children":31324},{},[31325,31330],{"type":150,"tag":158,"props":31326,"children":31327},{},[31328],{"type":155,"value":31329},"Sliding Window",{"type":155,"value":31331},"：保留最近的精细上下文",{"type":150,"tag":151,"props":31333,"children":31334},{},[31335],{"type":155,"value":31336},"这两个压缩率不是估的。Hugging Face 对 V4 的实现文档给出了默认值：CSA 是 4，HCA 是 128。为什么非要留一个 sliding window？因为压缩是有损的。CSA 压到 1\u002F4，HCA 压到 1\u002F128，最近的几个 token 一旦被压进 block，细节就再也找不回来了。而语言模型对紧接着的上下文又格外敏感。代词指谁、缩进几个空格、刚才那个变量叫什么名字、上一个 tool call 返回了什么。",{"type":150,"tag":151,"props":31338,"children":31339},{},[31340],{"type":155,"value":31341},"这些信息全都发生在最后几百个 token 里。",{"type":150,"tag":151,"props":31343,"children":31344},{},[31345,31347,31423,31425,31467,31469,31511,31513,31555],{"type":155,"value":31346},"所以最近的一段必须原样保留。三类 attention 不是三个可以互相替代的方案，而是三段分工：最近的原样看，中间挑着看，最远的粗着看。这个分工其实很符合直觉。人读一篇长文档也是这样：最后一段逐字读，中间扫小标题，开头的细节早忘了，只记得大意。于是 attention 不再是简单的 ",{"type":150,"tag":169,"props":31348,"children":31350},{"className":31349},[177],[31351,31380],{"type":150,"tag":169,"props":31352,"children":31354},{"className":31353},[182],[31355],{"type":150,"tag":185,"props":31356,"children":31357},{"xmlns":187},[31358],{"type":150,"tag":191,"props":31359,"children":31360},{},[31361,31376],{"type":150,"tag":195,"props":31362,"children":31363},{},[31364,31368,31372],{"type":150,"tag":405,"props":31365,"children":31366},{},[31367],{"type":155,"value":5591},{"type":150,"tag":417,"props":31369,"children":31370},{},[31371],{"type":155,"value":5554},{"type":150,"tag":405,"props":31373,"children":31374},{},[31375],{"type":155,"value":5591},{"type":150,"tag":223,"props":31377,"children":31378},{"encoding":225},[31379],{"type":155,"value":31144},{"type":150,"tag":169,"props":31381,"children":31383},{"className":31382,"ariaHidden":214},[232],[31384,31410],{"type":150,"tag":169,"props":31385,"children":31387},{"className":31386},[237],[31388,31392,31397,31401,31406],{"type":150,"tag":169,"props":31389,"children":31391},{"className":31390,"style":5644},[242],[],{"type":150,"tag":169,"props":31393,"children":31395},{"className":31394,"style":5780},[248,489],[31396],{"type":155,"value":5591},{"type":150,"tag":169,"props":31398,"children":31400},{"className":31399,"style":5167},[558],[],{"type":150,"tag":169,"props":31402,"children":31404},{"className":31403},[5172],[31405],{"type":155,"value":5554},{"type":150,"tag":169,"props":31407,"children":31409},{"className":31408,"style":5167},[558],[],{"type":150,"tag":169,"props":31411,"children":31413},{"className":31412},[237],[31414,31418],{"type":150,"tag":169,"props":31415,"children":31417},{"className":31416,"style":5854},[242],[],{"type":150,"tag":169,"props":31419,"children":31421},{"className":31420,"style":5780},[248,489],[31422],{"type":155,"value":5591},{"type":155,"value":31424},"，而是三种东西的叠加（记局部窗口大小为 ",{"type":150,"tag":169,"props":31426,"children":31428},{"className":31427},[177],[31429,31450],{"type":150,"tag":169,"props":31430,"children":31432},{"className":31431},[182],[31433],{"type":150,"tag":185,"props":31434,"children":31435},{"xmlns":187},[31436],{"type":150,"tag":191,"props":31437,"children":31438},{},[31439,31446],{"type":150,"tag":195,"props":31440,"children":31441},{},[31442],{"type":150,"tag":405,"props":31443,"children":31444},{},[31445],{"type":155,"value":3443},{"type":150,"tag":223,"props":31447,"children":31448},{"encoding":225},[31449],{"type":155,"value":3443},{"type":150,"tag":169,"props":31451,"children":31453},{"className":31452,"ariaHidden":214},[232],[31454],{"type":150,"tag":169,"props":31455,"children":31457},{"className":31456},[237],[31458,31462],{"type":150,"tag":169,"props":31459,"children":31461},{"className":31460,"style":5854},[242],[],{"type":150,"tag":169,"props":31463,"children":31465},{"className":31464,"style":3869},[248,489],[31466],{"type":155,"value":3443},{"type":155,"value":31468},"，每个 query 选中的 block 数为 ",{"type":150,"tag":169,"props":31470,"children":31472},{"className":31471},[177],[31473,31494],{"type":150,"tag":169,"props":31474,"children":31476},{"className":31475},[182],[31477],{"type":150,"tag":185,"props":31478,"children":31479},{"xmlns":187},[31480],{"type":150,"tag":191,"props":31481,"children":31482},{},[31483,31490],{"type":150,"tag":195,"props":31484,"children":31485},{},[31486],{"type":150,"tag":405,"props":31487,"children":31488},{},[31489],{"type":155,"value":24918},{"type":150,"tag":223,"props":31491,"children":31492},{"encoding":225},[31493],{"type":155,"value":24918},{"type":150,"tag":169,"props":31495,"children":31497},{"className":31496,"ariaHidden":214},[232],[31498],{"type":150,"tag":169,"props":31499,"children":31501},{"className":31500},[237],[31502,31506],{"type":150,"tag":169,"props":31503,"children":31505},{"className":31504,"style":5854},[242],[],{"type":150,"tag":169,"props":31507,"children":31509},{"className":31508,"style":27238},[248,489],[31510],{"type":155,"value":24918},{"type":155,"value":31512},"，压缩率为 ",{"type":150,"tag":169,"props":31514,"children":31516},{"className":31515},[177],[31517,31538],{"type":150,"tag":169,"props":31518,"children":31520},{"className":31519},[182],[31521],{"type":150,"tag":185,"props":31522,"children":31523},{"xmlns":187},[31524],{"type":150,"tag":191,"props":31525,"children":31526},{},[31527,31534],{"type":150,"tag":195,"props":31528,"children":31529},{},[31530],{"type":150,"tag":405,"props":31531,"children":31532},{},[31533],{"type":155,"value":20607},{"type":150,"tag":223,"props":31535,"children":31536},{"encoding":225},[31537],{"type":155,"value":20607},{"type":150,"tag":169,"props":31539,"children":31541},{"className":31540,"ariaHidden":214},[232],[31542],{"type":150,"tag":169,"props":31543,"children":31545},{"className":31544},[237],[31546,31550],{"type":150,"tag":169,"props":31547,"children":31549},{"className":31548,"style":1094},[242],[],{"type":150,"tag":169,"props":31551,"children":31553},{"className":31552,"style":6968},[248,489],[31554],{"type":155,"value":20607},{"type":155,"value":31556},"）：",{"type":150,"tag":169,"props":31558,"children":31560},{"className":31559},[172],[31561],{"type":150,"tag":169,"props":31562,"children":31564},{"className":31563},[177],[31565,31603],{"type":150,"tag":169,"props":31566,"children":31568},{"className":31567},[182],[31569],{"type":150,"tag":185,"props":31570,"children":31571},{"xmlns":187,"display":188},[31572],{"type":150,"tag":191,"props":31573,"children":31574},{},[31575,31598],{"type":150,"tag":195,"props":31576,"children":31577},{},[31578,31582,31586,31590,31593],{"type":150,"tag":405,"props":31579,"children":31580},{},[31581],{"type":155,"value":5591},{"type":150,"tag":417,"props":31583,"children":31584},{},[31585],{"type":155,"value":5554},{"type":150,"tag":405,"props":31587,"children":31588},{},[31589],{"type":155,"value":3443},{"type":150,"tag":558,"props":31591,"children":31592},{"width":3491},[],{"type":150,"tag":217,"props":31594,"children":31595},{},[31596],{"type":155,"value":31597},"（局部窗口）",{"type":150,"tag":223,"props":31599,"children":31600},{"encoding":225},[31601],{"type":155,"value":31602},"N \\times W \\quad \\text{（局部窗口）}",{"type":150,"tag":169,"props":31604,"children":31606},{"className":31605,"ariaHidden":214},[232],[31607,31633],{"type":150,"tag":169,"props":31608,"children":31610},{"className":31609},[237],[31611,31615,31620,31624,31629],{"type":150,"tag":169,"props":31612,"children":31614},{"className":31613,"style":5644},[242],[],{"type":150,"tag":169,"props":31616,"children":31618},{"className":31617,"style":5780},[248,489],[31619],{"type":155,"value":5591},{"type":150,"tag":169,"props":31621,"children":31623},{"className":31622,"style":5167},[558],[],{"type":150,"tag":169,"props":31625,"children":31627},{"className":31626},[5172],[31628],{"type":155,"value":5554},{"type":150,"tag":169,"props":31630,"children":31632},{"className":31631,"style":5167},[558],[],{"type":150,"tag":169,"props":31634,"children":31636},{"className":31635},[237],[31637,31641,31646,31650],{"type":150,"tag":169,"props":31638,"children":31640},{"className":31639,"style":5854},[242],[],{"type":150,"tag":169,"props":31642,"children":31644},{"className":31643,"style":3869},[248,489],[31645],{"type":155,"value":3443},{"type":150,"tag":169,"props":31647,"children":31649},{"className":31648,"style":4087},[558],[],{"type":150,"tag":169,"props":31651,"children":31653},{"className":31652},[248,155],[31654],{"type":150,"tag":169,"props":31655,"children":31657},{"className":31656},[248,297],[31658],{"type":155,"value":31597},{"type":150,"tag":169,"props":31660,"children":31662},{"className":31661},[172],[31663],{"type":150,"tag":169,"props":31664,"children":31666},{"className":31665},[177],[31667,31705],{"type":150,"tag":169,"props":31668,"children":31670},{"className":31669},[182],[31671],{"type":150,"tag":185,"props":31672,"children":31673},{"xmlns":187,"display":188},[31674],{"type":150,"tag":191,"props":31675,"children":31676},{},[31677,31700],{"type":150,"tag":195,"props":31678,"children":31679},{},[31680,31684,31688,31692,31695],{"type":150,"tag":405,"props":31681,"children":31682},{},[31683],{"type":155,"value":5591},{"type":150,"tag":417,"props":31685,"children":31686},{},[31687],{"type":155,"value":5554},{"type":150,"tag":405,"props":31689,"children":31690},{},[31691],{"type":155,"value":24918},{"type":150,"tag":558,"props":31693,"children":31694},{"width":3491},[],{"type":150,"tag":217,"props":31696,"children":31697},{},[31698],{"type":155,"value":31699},"（稀疏选中的 compressed blocks）",{"type":150,"tag":223,"props":31701,"children":31702},{"encoding":225},[31703],{"type":155,"value":31704},"N \\times B \\quad \\text{（稀疏选中的 compressed blocks）}",{"type":150,"tag":169,"props":31706,"children":31708},{"className":31707,"ariaHidden":214},[232],[31709,31735],{"type":150,"tag":169,"props":31710,"children":31712},{"className":31711},[237],[31713,31717,31722,31726,31731],{"type":150,"tag":169,"props":31714,"children":31716},{"className":31715,"style":5644},[242],[],{"type":150,"tag":169,"props":31718,"children":31720},{"className":31719,"style":5780},[248,489],[31721],{"type":155,"value":5591},{"type":150,"tag":169,"props":31723,"children":31725},{"className":31724,"style":5167},[558],[],{"type":150,"tag":169,"props":31727,"children":31729},{"className":31728},[5172],[31730],{"type":155,"value":5554},{"type":150,"tag":169,"props":31732,"children":31734},{"className":31733,"style":5167},[558],[],{"type":150,"tag":169,"props":31736,"children":31738},{"className":31737},[237],[31739,31743,31748,31752],{"type":150,"tag":169,"props":31740,"children":31742},{"className":31741,"style":1121},[242],[],{"type":150,"tag":169,"props":31744,"children":31746},{"className":31745,"style":27238},[248,489],[31747],{"type":155,"value":24918},{"type":150,"tag":169,"props":31749,"children":31751},{"className":31750,"style":4087},[558],[],{"type":150,"tag":169,"props":31753,"children":31755},{"className":31754},[248,155],[31756,31762,31768],{"type":150,"tag":169,"props":31757,"children":31759},{"className":31758},[248,297],[31760],{"type":155,"value":31761},"（稀疏选中的",{"type":150,"tag":169,"props":31763,"children":31765},{"className":31764},[248],[31766],{"type":155,"value":31767}," compressed blocks",{"type":150,"tag":169,"props":31769,"children":31771},{"className":31770},[248,297],[31772],{"type":155,"value":31773},"）",{"type":150,"tag":169,"props":31775,"children":31777},{"className":31776},[172],[31778],{"type":150,"tag":169,"props":31779,"children":31781},{"className":31780},[177],[31782,31827],{"type":150,"tag":169,"props":31783,"children":31785},{"className":31784},[182],[31786],{"type":150,"tag":185,"props":31787,"children":31788},{"xmlns":187,"display":188},[31789],{"type":150,"tag":191,"props":31790,"children":31791},{},[31792,31822],{"type":150,"tag":195,"props":31793,"children":31794},{},[31795,31799,31803,31814,31817],{"type":150,"tag":405,"props":31796,"children":31797},{},[31798],{"type":155,"value":5591},{"type":150,"tag":417,"props":31800,"children":31801},{},[31802],{"type":155,"value":5554},{"type":150,"tag":1952,"props":31804,"children":31805},{},[31806,31810],{"type":150,"tag":405,"props":31807,"children":31808},{},[31809],{"type":155,"value":5591},{"type":150,"tag":405,"props":31811,"children":31812},{},[31813],{"type":155,"value":20607},{"type":150,"tag":558,"props":31815,"children":31816},{"width":3491},[],{"type":150,"tag":217,"props":31818,"children":31819},{},[31820],{"type":155,"value":31821},"（高度压缩的 global branch）",{"type":150,"tag":223,"props":31823,"children":31824},{"encoding":225},[31825],{"type":155,"value":31826},"N \\times \\frac{N}{r} \\quad \\text{（高度压缩的 global branch）}",{"type":150,"tag":169,"props":31828,"children":31830},{"className":31829,"ariaHidden":214},[232],[31831,31857],{"type":150,"tag":169,"props":31832,"children":31834},{"className":31833},[237],[31835,31839,31844,31848,31853],{"type":150,"tag":169,"props":31836,"children":31838},{"className":31837,"style":5644},[242],[],{"type":150,"tag":169,"props":31840,"children":31842},{"className":31841,"style":5780},[248,489],[31843],{"type":155,"value":5591},{"type":150,"tag":169,"props":31845,"children":31847},{"className":31846,"style":5167},[558],[],{"type":150,"tag":169,"props":31849,"children":31851},{"className":31850},[5172],[31852],{"type":155,"value":5554},{"type":150,"tag":169,"props":31854,"children":31856},{"className":31855,"style":5167},[558],[],{"type":150,"tag":169,"props":31858,"children":31860},{"className":31859},[237],[31861,31866,31953,31957],{"type":150,"tag":169,"props":31862,"children":31865},{"className":31863,"style":31864},[242],"height:2.0463em;vertical-align:-0.686em;",[],{"type":150,"tag":169,"props":31867,"children":31869},{"className":31868},[248],[31870,31874,31949],{"type":150,"tag":169,"props":31871,"children":31873},{"className":31872},[1004,2660],[],{"type":150,"tag":169,"props":31875,"children":31877},{"className":31876},[1952],[31878],{"type":150,"tag":169,"props":31879,"children":31881},{"className":31880},[253,254],[31882,31938],{"type":150,"tag":169,"props":31883,"children":31885},{"className":31884},[259],[31886,31933],{"type":150,"tag":169,"props":31887,"children":31889},{"className":31888,"style":23685},[264],[31890,31906,31917],{"type":150,"tag":169,"props":31891,"children":31892},{"style":8422},[31893,31897],{"type":150,"tag":169,"props":31894,"children":31896},{"className":31895,"style":8427},[274],[],{"type":150,"tag":169,"props":31898,"children":31900},{"className":31899},[248],[31901],{"type":150,"tag":169,"props":31902,"children":31904},{"className":31903,"style":6968},[248,489],[31905],{"type":155,"value":20607},{"type":150,"tag":169,"props":31907,"children":31908},{"style":8440},[31909,31913],{"type":150,"tag":169,"props":31910,"children":31912},{"className":31911,"style":8427},[274],[],{"type":150,"tag":169,"props":31914,"children":31916},{"className":31915,"style":2982},[2981],[],{"type":150,"tag":169,"props":31918,"children":31919},{"style":8452},[31920,31924],{"type":150,"tag":169,"props":31921,"children":31923},{"className":31922,"style":8427},[274],[],{"type":150,"tag":169,"props":31925,"children":31927},{"className":31926},[248],[31928],{"type":150,"tag":169,"props":31929,"children":31931},{"className":31930,"style":5780},[248,489],[31932],{"type":155,"value":5591},{"type":150,"tag":169,"props":31934,"children":31936},{"className":31935},[355],[31937],{"type":155,"value":358},{"type":150,"tag":169,"props":31939,"children":31941},{"className":31940},[259],[31942],{"type":150,"tag":169,"props":31943,"children":31945},{"className":31944,"style":8484},[264],[31946],{"type":150,"tag":169,"props":31947,"children":31948},{},[],{"type":150,"tag":169,"props":31950,"children":31952},{"className":31951},[1015,2660],[],{"type":150,"tag":169,"props":31954,"children":31956},{"className":31955,"style":4087},[558],[],{"type":150,"tag":169,"props":31958,"children":31960},{"className":31959},[248,155],[31961,31967,31973],{"type":150,"tag":169,"props":31962,"children":31964},{"className":31963},[248,297],[31965],{"type":155,"value":31966},"（高度压缩的",{"type":150,"tag":169,"props":31968,"children":31970},{"className":31969},[248],[31971],{"type":155,"value":31972}," global branch",{"type":150,"tag":169,"props":31974,"children":31976},{"className":31975},[248,297],[31977],{"type":155,"value":31773},{"type":150,"tag":30755,"props":31979,"children":31981},{"id":31980},"复杂度没有变成-on",[31982],{"type":155,"value":31983},"复杂度没有变成 O(N)",{"type":150,"tag":151,"props":31985,"children":31986},{},[31987,31989,32031,32033,32137],{"type":155,"value":31988},"如果 HCA 那一路的 ",{"type":150,"tag":169,"props":31990,"children":31992},{"className":31991},[177],[31993,32014],{"type":150,"tag":169,"props":31994,"children":31996},{"className":31995},[182],[31997],{"type":150,"tag":185,"props":31998,"children":31999},{"xmlns":187},[32000],{"type":150,"tag":191,"props":32001,"children":32002},{},[32003,32010],{"type":150,"tag":195,"props":32004,"children":32005},{},[32006],{"type":150,"tag":405,"props":32007,"children":32008},{},[32009],{"type":155,"value":20607},{"type":150,"tag":223,"props":32011,"children":32012},{"encoding":225},[32013],{"type":155,"value":20607},{"type":150,"tag":169,"props":32015,"children":32017},{"className":32016,"ariaHidden":214},[232],[32018],{"type":150,"tag":169,"props":32019,"children":32021},{"className":32020},[237],[32022,32026],{"type":150,"tag":169,"props":32023,"children":32025},{"className":32024,"style":1094},[242],[],{"type":150,"tag":169,"props":32027,"children":32029},{"className":32028,"style":6968},[248,489],[32030],{"type":155,"value":20607},{"type":155,"value":32032}," 是个固定常数，那么它的理论渐进复杂度仍然带有 ",{"type":150,"tag":169,"props":32034,"children":32036},{"className":32035},[177],[32037,32074],{"type":150,"tag":169,"props":32038,"children":32040},{"className":32039},[182],[32041],{"type":150,"tag":185,"props":32042,"children":32043},{"xmlns":187},[32044],{"type":150,"tag":191,"props":32045,"children":32046},{},[32047,32069],{"type":150,"tag":195,"props":32048,"children":32049},{},[32050,32061,32065],{"type":150,"tag":746,"props":32051,"children":32052},{},[32053,32057],{"type":150,"tag":405,"props":32054,"children":32055},{},[32056],{"type":155,"value":5591},{"type":150,"tag":411,"props":32058,"children":32059},{},[32060],{"type":155,"value":433},{"type":150,"tag":405,"props":32062,"children":32063},{"mathvariant":1796},[32064],{"type":155,"value":4914},{"type":150,"tag":405,"props":32066,"children":32067},{},[32068],{"type":155,"value":20607},{"type":150,"tag":223,"props":32070,"children":32071},{"encoding":225},[32072],{"type":155,"value":32073},"N^2\u002Fr",{"type":150,"tag":169,"props":32075,"children":32077},{"className":32076,"ariaHidden":214},[232],[32078],{"type":150,"tag":169,"props":32079,"children":32081},{"className":32080},[237],[32082,32086,32127,32132],{"type":150,"tag":169,"props":32083,"children":32085},{"className":32084,"style":6963},[242],[],{"type":150,"tag":169,"props":32087,"children":32089},{"className":32088},[248],[32090,32095],{"type":150,"tag":169,"props":32091,"children":32093},{"className":32092,"style":5780},[248,489],[32094],{"type":155,"value":5591},{"type":150,"tag":169,"props":32096,"children":32098},{"className":32097},[495],[32099],{"type":150,"tag":169,"props":32100,"children":32102},{"className":32101},[253],[32103],{"type":150,"tag":169,"props":32104,"children":32106},{"className":32105},[259],[32107],{"type":150,"tag":169,"props":32108,"children":32110},{"className":32109,"style":7005},[264],[32111],{"type":150,"tag":169,"props":32112,"children":32113},{"style":888},[32114,32118],{"type":150,"tag":169,"props":32115,"children":32117},{"className":32116,"style":517},[274],[],{"type":150,"tag":169,"props":32119,"children":32121},{"className":32120},[522,523,524,525],[32122],{"type":150,"tag":169,"props":32123,"children":32125},{"className":32124},[248,525],[32126],{"type":155,"value":433},{"type":150,"tag":169,"props":32128,"children":32130},{"className":32129},[248],[32131],{"type":155,"value":4914},{"type":150,"tag":169,"props":32133,"children":32135},{"className":32134,"style":6968},[248,489],[32136],{"type":155,"value":20607},{"type":155,"value":904},{"type":150,"tag":151,"props":32139,"children":32140},{},[32141],{"type":150,"tag":158,"props":32142,"children":32143},{},[32144,32146,32216],{"type":155,"value":32145},"并没有数学意义上 magically 变成 ",{"type":150,"tag":169,"props":32147,"children":32149},{"className":32148},[177],[32150,32184],{"type":150,"tag":169,"props":32151,"children":32153},{"className":32152},[182],[32154],{"type":150,"tag":185,"props":32155,"children":32156},{"xmlns":187},[32157],{"type":150,"tag":191,"props":32158,"children":32159},{},[32160,32179],{"type":150,"tag":195,"props":32161,"children":32162},{},[32163,32167,32171,32175],{"type":150,"tag":405,"props":32164,"children":32165},{},[32166],{"type":155,"value":6923},{"type":150,"tag":417,"props":32168,"children":32169},{"stretchy":207},[32170],{"type":155,"value":972},{"type":150,"tag":405,"props":32172,"children":32173},{},[32174],{"type":155,"value":5591},{"type":150,"tag":417,"props":32176,"children":32177},{"stretchy":207},[32178],{"type":155,"value":982},{"type":150,"tag":223,"props":32180,"children":32181},{"encoding":225},[32182],{"type":155,"value":32183},"O(N)",{"type":150,"tag":169,"props":32185,"children":32187},{"className":32186,"ariaHidden":214},[232],[32188],{"type":150,"tag":169,"props":32189,"children":32191},{"className":32190},[237],[32192,32196,32201,32206,32211],{"type":150,"tag":169,"props":32193,"children":32195},{"className":32194,"style":999},[242],[],{"type":150,"tag":169,"props":32197,"children":32199},{"className":32198,"style":6968},[248,489],[32200],{"type":155,"value":6923},{"type":150,"tag":169,"props":32202,"children":32204},{"className":32203},[1004],[32205],{"type":155,"value":972},{"type":150,"tag":169,"props":32207,"children":32209},{"className":32208,"style":5780},[248,489],[32210],{"type":155,"value":5591},{"type":150,"tag":169,"props":32212,"children":32214},{"className":32213},[1015],[32215],{"type":155,"value":982},{"type":155,"value":904},{"type":150,"tag":151,"props":32218,"children":32219},{},[32220],{"type":155,"value":32221},"但工程量级已经差很多。而 CSA 的 sparse branch 更接近：",{"type":150,"tag":169,"props":32223,"children":32225},{"className":32224},[172],[32226],{"type":150,"tag":169,"props":32227,"children":32229},{"className":32228},[177],[32230,32288],{"type":150,"tag":169,"props":32231,"children":32233},{"className":32232},[182],[32234],{"type":150,"tag":185,"props":32235,"children":32236},{"xmlns":187,"display":188},[32237],{"type":150,"tag":191,"props":32238,"children":32239},{},[32240,32283],{"type":150,"tag":195,"props":32241,"children":32242},{},[32243,32247,32251,32255,32259,32263,32267,32270,32274,32279],{"type":150,"tag":405,"props":32244,"children":32245},{},[32246],{"type":155,"value":6923},{"type":150,"tag":417,"props":32248,"children":32249},{"stretchy":207},[32250],{"type":155,"value":972},{"type":150,"tag":405,"props":32252,"children":32253},{},[32254],{"type":155,"value":5591},{"type":150,"tag":405,"props":32256,"children":32257},{},[32258],{"type":155,"value":24918},{"type":150,"tag":417,"props":32260,"children":32261},{"stretchy":207},[32262],{"type":155,"value":982},{"type":150,"tag":417,"props":32264,"children":32265},{"separator":214},[32266],{"type":155,"value":421},{"type":150,"tag":558,"props":32268,"children":32269},{"width":3491},[],{"type":150,"tag":405,"props":32271,"children":32272},{},[32273],{"type":155,"value":24918},{"type":150,"tag":417,"props":32275,"children":32276},{},[32277],{"type":155,"value":32278},"≪",{"type":150,"tag":405,"props":32280,"children":32281},{},[32282],{"type":155,"value":5591},{"type":150,"tag":223,"props":32284,"children":32285},{"encoding":225},[32286],{"type":155,"value":32287},"O(NB), \\quad B \\ll N",{"type":150,"tag":169,"props":32289,"children":32291},{"className":32290,"ariaHidden":214},[232],[32292,32356],{"type":150,"tag":169,"props":32293,"children":32295},{"className":32294},[237],[32296,32300,32305,32310,32315,32320,32325,32330,32334,32338,32343,32347,32352],{"type":150,"tag":169,"props":32297,"children":32299},{"className":32298,"style":999},[242],[],{"type":150,"tag":169,"props":32301,"children":32303},{"className":32302,"style":6968},[248,489],[32304],{"type":155,"value":6923},{"type":150,"tag":169,"props":32306,"children":32308},{"className":32307},[1004],[32309],{"type":155,"value":972},{"type":150,"tag":169,"props":32311,"children":32313},{"className":32312,"style":5780},[248,489],[32314],{"type":155,"value":5591},{"type":150,"tag":169,"props":32316,"children":32318},{"className":32317,"style":27238},[248,489],[32319],{"type":155,"value":24918},{"type":150,"tag":169,"props":32321,"children":32323},{"className":32322},[1015],[32324],{"type":155,"value":982},{"type":150,"tag":169,"props":32326,"children":32328},{"className":32327},[552],[32329],{"type":155,"value":421},{"type":150,"tag":169,"props":32331,"children":32333},{"className":32332,"style":4087},[558],[],{"type":150,"tag":169,"props":32335,"children":32337},{"className":32336,"style":559},[558],[],{"type":150,"tag":169,"props":32339,"children":32341},{"className":32340,"style":27238},[248,489],[32342],{"type":155,"value":24918},{"type":150,"tag":169,"props":32344,"children":32346},{"className":32345,"style":839},[558],[],{"type":150,"tag":169,"props":32348,"children":32350},{"className":32349},[844],[32351],{"type":155,"value":32278},{"type":150,"tag":169,"props":32353,"children":32355},{"className":32354,"style":839},[558],[],{"type":150,"tag":169,"props":32357,"children":32359},{"className":32358},[237],[32360,32364],{"type":150,"tag":169,"props":32361,"children":32363},{"className":32362,"style":5854},[242],[],{"type":150,"tag":169,"props":32365,"children":32367},{"className":32366,"style":5780},[248,489],[32368],{"type":155,"value":5591},{"type":150,"tag":151,"props":32370,"children":32371},{},[32372],{"type":155,"value":32373},"这才是 V4 真正攻击 quadratic attention 的部分。所以\"V4 把 attention 变成线性了\"这种说法，把两件事混成了一件。严格结论和工程结论要分开讲。",{"type":150,"tag":372,"props":32375,"children":32377},{"id":32376},"linear-attention-与-ssm",[32378],{"type":155,"value":32379},"Linear Attention 与 SSM",{"type":150,"tag":151,"props":32381,"children":32382},{},[32383,32385,32390],{"type":155,"value":32384},"前面所有方案都建立在同一个前提上：KV 要保存下来。那换个问题。",{"type":150,"tag":158,"props":32386,"children":32387},{},[32388],{"type":155,"value":32389},"如果 KV 这么贵，为什么一定要保存它？",{"type":155,"value":32391}," RNN 的答案是最极端的：",{"type":150,"tag":169,"props":32393,"children":32395},{"className":32394},[172],[32396],{"type":150,"tag":169,"props":32397,"children":32399},{"className":32398},[177],[32400,32517],{"type":150,"tag":169,"props":32401,"children":32403},{"className":32402},[182],[32404],{"type":150,"tag":185,"props":32405,"children":32406},{"xmlns":187,"display":188},[32407],{"type":150,"tag":191,"props":32408,"children":32409},{},[32410,32512],{"type":150,"tag":195,"props":32411,"children":32412},{},[32413,32424,32428,32433,32437,32441,32452,32463,32467,32478,32500,32504,32508],{"type":150,"tag":401,"props":32414,"children":32415},{},[32416,32420],{"type":150,"tag":405,"props":32417,"children":32418},{},[32419],{"type":155,"value":926},{"type":150,"tag":405,"props":32421,"children":32422},{},[32423],{"type":155,"value":1213},{"type":150,"tag":417,"props":32425,"children":32426},{},[32427],{"type":155,"value":1045},{"type":150,"tag":405,"props":32429,"children":32430},{},[32431],{"type":155,"value":32432},"tanh",{"type":150,"tag":417,"props":32434,"children":32435},{},[32436],{"type":155,"value":1991},{"type":150,"tag":417,"props":32438,"children":32439},{"stretchy":207},[32440],{"type":155,"value":972},{"type":150,"tag":401,"props":32442,"children":32443},{},[32444,32448],{"type":150,"tag":405,"props":32445,"children":32446},{},[32447],{"type":155,"value":3443},{"type":150,"tag":405,"props":32449,"children":32450},{},[32451],{"type":155,"value":409},{"type":150,"tag":401,"props":32453,"children":32454},{},[32455,32459],{"type":150,"tag":405,"props":32456,"children":32457},{},[32458],{"type":155,"value":409},{"type":150,"tag":405,"props":32460,"children":32461},{},[32462],{"type":155,"value":1213},{"type":150,"tag":417,"props":32464,"children":32465},{},[32466],{"type":155,"value":5136},{"type":150,"tag":401,"props":32468,"children":32469},{},[32470,32474],{"type":150,"tag":405,"props":32471,"children":32472},{},[32473],{"type":155,"value":3443},{"type":150,"tag":405,"props":32475,"children":32476},{},[32477],{"type":155,"value":926},{"type":150,"tag":401,"props":32479,"children":32480},{},[32481,32485],{"type":150,"tag":405,"props":32482,"children":32483},{},[32484],{"type":155,"value":926},{"type":150,"tag":195,"props":32486,"children":32487},{},[32488,32492,32496],{"type":150,"tag":405,"props":32489,"children":32490},{},[32491],{"type":155,"value":1213},{"type":150,"tag":417,"props":32493,"children":32494},{},[32495],{"type":155,"value":18989},{"type":150,"tag":411,"props":32497,"children":32498},{},[32499],{"type":155,"value":415},{"type":150,"tag":417,"props":32501,"children":32502},{},[32503],{"type":155,"value":5136},{"type":150,"tag":405,"props":32505,"children":32506},{},[32507],{"type":155,"value":22264},{"type":150,"tag":417,"props":32509,"children":32510},{"stretchy":207},[32511],{"type":155,"value":982},{"type":150,"tag":223,"props":32513,"children":32514},{"encoding":225},[32515],{"type":155,"value":32516},"h_t = \\tanh(W_x x_t + W_h h_{t-1} + b)",{"type":150,"tag":169,"props":32518,"children":32520},{"className":32519,"ariaHidden":214},[232],[32521,32599,32744,32894],{"type":150,"tag":169,"props":32522,"children":32524},{"className":32523},[237],[32525,32529,32586,32590,32595],{"type":150,"tag":169,"props":32526,"children":32528},{"className":32527,"style":4935},[242],[],{"type":150,"tag":169,"props":32530,"children":32532},{"className":32531},[248],[32533,32538],{"type":150,"tag":169,"props":32534,"children":32536},{"className":32535},[248,489],[32537],{"type":155,"value":926},{"type":150,"tag":169,"props":32539,"children":32541},{"className":32540},[495],[32542],{"type":150,"tag":169,"props":32543,"children":32545},{"className":32544},[253,254],[32546,32575],{"type":150,"tag":169,"props":32547,"children":32549},{"className":32548},[259],[32550,32570],{"type":150,"tag":169,"props":32551,"children":32553},{"className":32552,"style":1369},[264],[32554],{"type":150,"tag":169,"props":32555,"children":32556},{"style":512},[32557,32561],{"type":150,"tag":169,"props":32558,"children":32560},{"className":32559,"style":517},[274],[],{"type":150,"tag":169,"props":32562,"children":32564},{"className":32563},[522,523,524,525],[32565],{"type":150,"tag":169,"props":32566,"children":32568},{"className":32567},[248,489,525],[32569],{"type":155,"value":1213},{"type":150,"tag":169,"props":32571,"children":32573},{"className":32572},[355],[32574],{"type":155,"value":358},{"type":150,"tag":169,"props":32576,"children":32578},{"className":32577},[259],[32579],{"type":150,"tag":169,"props":32580,"children":32582},{"className":32581,"style":544},[264],[32583],{"type":150,"tag":169,"props":32584,"children":32585},{},[],{"type":150,"tag":169,"props":32587,"children":32589},{"className":32588,"style":839},[558],[],{"type":150,"tag":169,"props":32591,"children":32593},{"className":32592},[844],[32594],{"type":155,"value":1045},{"type":150,"tag":169,"props":32596,"children":32598},{"className":32597,"style":839},[558],[],{"type":150,"tag":169,"props":32600,"children":32602},{"className":32601},[237],[32603,32607,32612,32617,32674,32731,32735,32740],{"type":150,"tag":169,"props":32604,"children":32606},{"className":32605,"style":999},[242],[],{"type":150,"tag":169,"props":32608,"children":32610},{"className":32609},[2695],[32611],{"type":155,"value":32432},{"type":150,"tag":169,"props":32613,"children":32615},{"className":32614},[1004],[32616],{"type":155,"value":972},{"type":150,"tag":169,"props":32618,"children":32620},{"className":32619},[248],[32621,32626],{"type":150,"tag":169,"props":32622,"children":32624},{"className":32623,"style":3869},[248,489],[32625],{"type":155,"value":3443},{"type":150,"tag":169,"props":32627,"children":32629},{"className":32628},[495],[32630],{"type":150,"tag":169,"props":32631,"children":32633},{"className":32632},[253,254],[32634,32663],{"type":150,"tag":169,"props":32635,"children":32637},{"className":32636},[259],[32638,32658],{"type":150,"tag":169,"props":32639,"children":32641},{"className":32640,"style":5039},[264],[32642],{"type":150,"tag":169,"props":32643,"children":32644},{"style":11307},[32645,32649],{"type":150,"tag":169,"props":32646,"children":32648},{"className":32647,"style":517},[274],[],{"type":150,"tag":169,"props":32650,"children":32652},{"className":32651},[522,523,524,525],[32653],{"type":150,"tag":169,"props":32654,"children":32656},{"className":32655},[248,489,525],[32657],{"type":155,"value":409},{"type":150,"tag":169,"props":32659,"children":32661},{"className":32660},[355],[32662],{"type":155,"value":358},{"type":150,"tag":169,"props":32664,"children":32666},{"className":32665},[259],[32667],{"type":150,"tag":169,"props":32668,"children":32670},{"className":32669,"style":544},[264],[32671],{"type":150,"tag":169,"props":32672,"children":32673},{},[],{"type":150,"tag":169,"props":32675,"children":32677},{"className":32676},[248],[32678,32683],{"type":150,"tag":169,"props":32679,"children":32681},{"className":32680},[248,489],[32682],{"type":155,"value":409},{"type":150,"tag":169,"props":32684,"children":32686},{"className":32685},[495],[32687],{"type":150,"tag":169,"props":32688,"children":32690},{"className":32689},[253,254],[32691,32720],{"type":150,"tag":169,"props":32692,"children":32694},{"className":32693},[259],[32695,32715],{"type":150,"tag":169,"props":32696,"children":32698},{"className":32697,"style":1369},[264],[32699],{"type":150,"tag":169,"props":32700,"children":32701},{"style":512},[32702,32706],{"type":150,"tag":169,"props":32703,"children":32705},{"className":32704,"style":517},[274],[],{"type":150,"tag":169,"props":32707,"children":32709},{"className":32708},[522,523,524,525],[32710],{"type":150,"tag":169,"props":32711,"children":32713},{"className":32712},[248,489,525],[32714],{"type":155,"value":1213},{"type":150,"tag":169,"props":32716,"children":32718},{"className":32717},[355],[32719],{"type":155,"value":358},{"type":150,"tag":169,"props":32721,"children":32723},{"className":32722},[259],[32724],{"type":150,"tag":169,"props":32725,"children":32727},{"className":32726,"style":544},[264],[32728],{"type":150,"tag":169,"props":32729,"children":32730},{},[],{"type":150,"tag":169,"props":32732,"children":32734},{"className":32733,"style":5167},[558],[],{"type":150,"tag":169,"props":32736,"children":32738},{"className":32737},[5172],[32739],{"type":155,"value":5136},{"type":150,"tag":169,"props":32741,"children":32743},{"className":32742,"style":5167},[558],[],{"type":150,"tag":169,"props":32745,"children":32747},{"className":32746},[237],[32748,32753,32810,32881,32885,32890],{"type":150,"tag":169,"props":32749,"children":32752},{"className":32750,"style":32751},[242],"height:0.9028em;vertical-align:-0.2083em;",[],{"type":150,"tag":169,"props":32754,"children":32756},{"className":32755},[248],[32757,32762],{"type":150,"tag":169,"props":32758,"children":32760},{"className":32759,"style":3869},[248,489],[32761],{"type":155,"value":3443},{"type":150,"tag":169,"props":32763,"children":32765},{"className":32764},[495],[32766],{"type":150,"tag":169,"props":32767,"children":32769},{"className":32768},[253,254],[32770,32799],{"type":150,"tag":169,"props":32771,"children":32773},{"className":32772},[259],[32774,32794],{"type":150,"tag":169,"props":32775,"children":32777},{"className":32776,"style":674},[264],[32778],{"type":150,"tag":169,"props":32779,"children":32780},{"style":11307},[32781,32785],{"type":150,"tag":169,"props":32782,"children":32784},{"className":32783,"style":517},[274],[],{"type":150,"tag":169,"props":32786,"children":32788},{"className":32787},[522,523,524,525],[32789],{"type":150,"tag":169,"props":32790,"children":32792},{"className":32791},[248,489,525],[32793],{"type":155,"value":926},{"type":150,"tag":169,"props":32795,"children":32797},{"className":32796},[355],[32798],{"type":155,"value":358},{"type":150,"tag":169,"props":32800,"children":32802},{"className":32801},[259],[32803],{"type":150,"tag":169,"props":32804,"children":32806},{"className":32805,"style":544},[264],[32807],{"type":150,"tag":169,"props":32808,"children":32809},{},[],{"type":150,"tag":169,"props":32811,"children":32813},{"className":32812},[248],[32814,32819],{"type":150,"tag":169,"props":32815,"children":32817},{"className":32816},[248,489],[32818],{"type":155,"value":926},{"type":150,"tag":169,"props":32820,"children":32822},{"className":32821},[495],[32823],{"type":150,"tag":169,"props":32824,"children":32826},{"className":32825},[253,254],[32827,32870],{"type":150,"tag":169,"props":32828,"children":32830},{"className":32829},[259],[32831,32865],{"type":150,"tag":169,"props":32832,"children":32834},{"className":32833,"style":508},[264],[32835],{"type":150,"tag":169,"props":32836,"children":32837},{"style":512},[32838,32842],{"type":150,"tag":169,"props":32839,"children":32841},{"className":32840,"style":517},[274],[],{"type":150,"tag":169,"props":32843,"children":32845},{"className":32844},[522,523,524,525],[32846],{"type":150,"tag":169,"props":32847,"children":32849},{"className":32848},[248,525],[32850,32855,32860],{"type":150,"tag":169,"props":32851,"children":32853},{"className":32852},[248,489,525],[32854],{"type":155,"value":1213},{"type":150,"tag":169,"props":32856,"children":32858},{"className":32857},[5172,525],[32859],{"type":155,"value":18989},{"type":150,"tag":169,"props":32861,"children":32863},{"className":32862},[248,525],[32864],{"type":155,"value":415},{"type":150,"tag":169,"props":32866,"children":32868},{"className":32867},[355],[32869],{"type":155,"value":358},{"type":150,"tag":169,"props":32871,"children":32873},{"className":32872},[259],[32874],{"type":150,"tag":169,"props":32875,"children":32877},{"className":32876,"style":19957},[264],[32878],{"type":150,"tag":169,"props":32879,"children":32880},{},[],{"type":150,"tag":169,"props":32882,"children":32884},{"className":32883,"style":5167},[558],[],{"type":150,"tag":169,"props":32886,"children":32888},{"className":32887},[5172],[32889],{"type":155,"value":5136},{"type":150,"tag":169,"props":32891,"children":32893},{"className":32892,"style":5167},[558],[],{"type":150,"tag":169,"props":32895,"children":32897},{"className":32896},[237],[32898,32902,32907],{"type":150,"tag":169,"props":32899,"children":32901},{"className":32900,"style":999},[242],[],{"type":150,"tag":169,"props":32903,"children":32905},{"className":32904},[248,489],[32906],{"type":155,"value":22264},{"type":150,"tag":169,"props":32908,"children":32910},{"className":32909},[1015],[32911],{"type":155,"value":982},{"type":150,"tag":151,"props":32913,"children":32914},{},[32915],{"type":155,"value":32916},"过去全部信息压进一个固定大小的状态。不保存 KV，只保存状态。代价是长期记忆很差，而且不能并行。LSTM \u002F GRU 用门控改善了记忆，但没根治，也仍然不能并行。然后 Linear Attention 出现了：",{"type":150,"tag":169,"props":32918,"children":32920},{"className":32919},[172],[32921],{"type":150,"tag":169,"props":32922,"children":32924},{"className":32923},[177],[32925,33024],{"type":150,"tag":169,"props":32926,"children":32928},{"className":32927},[182],[32929],{"type":150,"tag":185,"props":32930,"children":32931},{"xmlns":187,"display":188},[32932],{"type":150,"tag":191,"props":32933,"children":32934},{},[32935,33019],{"type":150,"tag":195,"props":32936,"children":32937},{},[32938,32950,32954,32976,32980,32985,32989,33000,33004],{"type":150,"tag":401,"props":32939,"children":32940},{},[32941,32946],{"type":150,"tag":405,"props":32942,"children":32943},{},[32944],{"type":155,"value":32945},"S",{"type":150,"tag":405,"props":32947,"children":32948},{},[32949],{"type":155,"value":1213},{"type":150,"tag":417,"props":32951,"children":32952},{},[32953],{"type":155,"value":1045},{"type":150,"tag":401,"props":32955,"children":32956},{},[32957,32961],{"type":150,"tag":405,"props":32958,"children":32959},{},[32960],{"type":155,"value":32945},{"type":150,"tag":195,"props":32962,"children":32963},{},[32964,32968,32972],{"type":150,"tag":405,"props":32965,"children":32966},{},[32967],{"type":155,"value":1213},{"type":150,"tag":417,"props":32969,"children":32970},{},[32971],{"type":155,"value":18989},{"type":150,"tag":411,"props":32973,"children":32974},{},[32975],{"type":155,"value":415},{"type":150,"tag":417,"props":32977,"children":32978},{},[32979],{"type":155,"value":5136},{"type":150,"tag":405,"props":32981,"children":32982},{},[32983],{"type":155,"value":32984},"ϕ",{"type":150,"tag":417,"props":32986,"children":32987},{"stretchy":207},[32988],{"type":155,"value":972},{"type":150,"tag":401,"props":32990,"children":32991},{},[32992,32996],{"type":150,"tag":405,"props":32993,"children":32994},{},[32995],{"type":155,"value":1876},{"type":150,"tag":405,"props":32997,"children":32998},{},[32999],{"type":155,"value":1213},{"type":150,"tag":417,"props":33001,"children":33002},{"stretchy":207},[33003],{"type":155,"value":982},{"type":150,"tag":1224,"props":33005,"children":33006},{},[33007,33011,33015],{"type":150,"tag":405,"props":33008,"children":33009},{},[33010],{"type":155,"value":1915},{"type":150,"tag":405,"props":33012,"children":33013},{},[33014],{"type":155,"value":1213},{"type":150,"tag":405,"props":33016,"children":33017},{"mathvariant":1796},[33018],{"type":155,"value":2052},{"type":150,"tag":223,"props":33020,"children":33021},{"encoding":225},[33022],{"type":155,"value":33023},"S_t = S_{t-1} + \\phi(k_t) v_t^\\top",{"type":150,"tag":169,"props":33025,"children":33027},{"className":33026,"ariaHidden":214},[232],[33028,33108,33201],{"type":150,"tag":169,"props":33029,"children":33031},{"className":33030},[237],[33032,33036,33095,33099,33104],{"type":150,"tag":169,"props":33033,"children":33035},{"className":33034,"style":6364},[242],[],{"type":150,"tag":169,"props":33037,"children":33039},{"className":33038},[248],[33040,33046],{"type":150,"tag":169,"props":33041,"children":33044},{"className":33042,"style":33043},[248,489],"margin-right:0.0576em;",[33045],{"type":155,"value":32945},{"type":150,"tag":169,"props":33047,"children":33049},{"className":33048},[495],[33050],{"type":150,"tag":169,"props":33051,"children":33053},{"className":33052},[253,254],[33054,33084],{"type":150,"tag":169,"props":33055,"children":33057},{"className":33056},[259],[33058,33079],{"type":150,"tag":169,"props":33059,"children":33061},{"className":33060,"style":1369},[264],[33062],{"type":150,"tag":169,"props":33063,"children":33065},{"style":33064},"top:-2.55em;margin-left:-0.0576em;margin-right:0.05em;",[33066,33070],{"type":150,"tag":169,"props":33067,"children":33069},{"className":33068,"style":517},[274],[],{"type":150,"tag":169,"props":33071,"children":33073},{"className":33072},[522,523,524,525],[33074],{"type":150,"tag":169,"props":33075,"children":33077},{"className":33076},[248,489,525],[33078],{"type":155,"value":1213},{"type":150,"tag":169,"props":33080,"children":33082},{"className":33081},[355],[33083],{"type":155,"value":358},{"type":150,"tag":169,"props":33085,"children":33087},{"className":33086},[259],[33088],{"type":150,"tag":169,"props":33089,"children":33091},{"className":33090,"style":544},[264],[33092],{"type":150,"tag":169,"props":33093,"children":33094},{},[],{"type":150,"tag":169,"props":33096,"children":33098},{"className":33097,"style":839},[558],[],{"type":150,"tag":169,"props":33100,"children":33102},{"className":33101},[844],[33103],{"type":155,"value":1045},{"type":150,"tag":169,"props":33105,"children":33107},{"className":33106,"style":839},[558],[],{"type":150,"tag":169,"props":33109,"children":33111},{"className":33110},[237],[33112,33117,33188,33192,33197],{"type":150,"tag":169,"props":33113,"children":33116},{"className":33114,"style":33115},[242],"height:0.8917em;vertical-align:-0.2083em;",[],{"type":150,"tag":169,"props":33118,"children":33120},{"className":33119},[248],[33121,33126],{"type":150,"tag":169,"props":33122,"children":33124},{"className":33123,"style":33043},[248,489],[33125],{"type":155,"value":32945},{"type":150,"tag":169,"props":33127,"children":33129},{"className":33128},[495],[33130],{"type":150,"tag":169,"props":33131,"children":33133},{"className":33132},[253,254],[33134,33177],{"type":150,"tag":169,"props":33135,"children":33137},{"className":33136},[259],[33138,33172],{"type":150,"tag":169,"props":33139,"children":33141},{"className":33140,"style":508},[264],[33142],{"type":150,"tag":169,"props":33143,"children":33144},{"style":33064},[33145,33149],{"type":150,"tag":169,"props":33146,"children":33148},{"className":33147,"style":517},[274],[],{"type":150,"tag":169,"props":33150,"children":33152},{"className":33151},[522,523,524,525],[33153],{"type":150,"tag":169,"props":33154,"children":33156},{"className":33155},[248,525],[33157,33162,33167],{"type":150,"tag":169,"props":33158,"children":33160},{"className":33159},[248,489,525],[33161],{"type":155,"value":1213},{"type":150,"tag":169,"props":33163,"children":33165},{"className":33164},[5172,525],[33166],{"type":155,"value":18989},{"type":150,"tag":169,"props":33168,"children":33170},{"className":33169},[248,525],[33171],{"type":155,"value":415},{"type":150,"tag":169,"props":33173,"children":33175},{"className":33174},[355],[33176],{"type":155,"value":358},{"type":150,"tag":169,"props":33178,"children":33180},{"className":33179},[259],[33181],{"type":150,"tag":169,"props":33182,"children":33184},{"className":33183,"style":19957},[264],[33185],{"type":150,"tag":169,"props":33186,"children":33187},{},[],{"type":150,"tag":169,"props":33189,"children":33191},{"className":33190,"style":5167},[558],[],{"type":150,"tag":169,"props":33193,"children":33195},{"className":33194},[5172],[33196],{"type":155,"value":5136},{"type":150,"tag":169,"props":33198,"children":33200},{"className":33199,"style":5167},[558],[],{"type":150,"tag":169,"props":33202,"children":33204},{"className":33203},[237],[33205,33210,33215,33220,33277,33282],{"type":150,"tag":169,"props":33206,"children":33209},{"className":33207,"style":33208},[242],"height:1.1491em;vertical-align:-0.25em;",[],{"type":150,"tag":169,"props":33211,"children":33213},{"className":33212},[248,489],[33214],{"type":155,"value":32984},{"type":150,"tag":169,"props":33216,"children":33218},{"className":33217},[1004],[33219],{"type":155,"value":972},{"type":150,"tag":169,"props":33221,"children":33223},{"className":33222},[248],[33224,33229],{"type":150,"tag":169,"props":33225,"children":33227},{"className":33226,"style":2427},[248,489],[33228],{"type":155,"value":1876},{"type":150,"tag":169,"props":33230,"children":33232},{"className":33231},[495],[33233],{"type":150,"tag":169,"props":33234,"children":33236},{"className":33235},[253,254],[33237,33266],{"type":150,"tag":169,"props":33238,"children":33240},{"className":33239},[259],[33241,33261],{"type":150,"tag":169,"props":33242,"children":33244},{"className":33243,"style":1369},[264],[33245],{"type":150,"tag":169,"props":33246,"children":33247},{"style":14350},[33248,33252],{"type":150,"tag":169,"props":33249,"children":33251},{"className":33250,"style":517},[274],[],{"type":150,"tag":169,"props":33253,"children":33255},{"className":33254},[522,523,524,525],[33256],{"type":150,"tag":169,"props":33257,"children":33259},{"className":33258},[248,489,525],[33260],{"type":155,"value":1213},{"type":150,"tag":169,"props":33262,"children":33264},{"className":33263},[355],[33265],{"type":155,"value":358},{"type":150,"tag":169,"props":33267,"children":33269},{"className":33268},[259],[33270],{"type":150,"tag":169,"props":33271,"children":33273},{"className":33272,"style":544},[264],[33274],{"type":150,"tag":169,"props":33275,"children":33276},{},[],{"type":150,"tag":169,"props":33278,"children":33280},{"className":33279},[1015],[33281],{"type":155,"value":982},{"type":150,"tag":169,"props":33283,"children":33285},{"className":33284},[248],[33286,33291],{"type":150,"tag":169,"props":33287,"children":33289},{"className":33288,"style":2329},[248,489],[33290],{"type":155,"value":1915},{"type":150,"tag":169,"props":33292,"children":33294},{"className":33293},[495],[33295],{"type":150,"tag":169,"props":33296,"children":33298},{"className":33297},[253,254],[33299,33345],{"type":150,"tag":169,"props":33300,"children":33302},{"className":33301},[259],[33303,33340],{"type":150,"tag":169,"props":33304,"children":33306},{"className":33305,"style":3999},[264],[33307,33324],{"type":150,"tag":169,"props":33308,"children":33310},{"style":33309},"top:-2.453em;margin-left:-0.0359em;margin-right:0.05em;",[33311,33315],{"type":150,"tag":169,"props":33312,"children":33314},{"className":33313,"style":517},[274],[],{"type":150,"tag":169,"props":33316,"children":33318},{"className":33317},[522,523,524,525],[33319],{"type":150,"tag":169,"props":33320,"children":33322},{"className":33321},[248,489,525],[33323],{"type":155,"value":1213},{"type":150,"tag":169,"props":33325,"children":33326},{"style":3908},[33327,33331],{"type":150,"tag":169,"props":33328,"children":33330},{"className":33329,"style":517},[274],[],{"type":150,"tag":169,"props":33332,"children":33334},{"className":33333},[522,523,524,525],[33335],{"type":150,"tag":169,"props":33336,"children":33338},{"className":33337},[248,525],[33339],{"type":155,"value":2052},{"type":150,"tag":169,"props":33341,"children":33343},{"className":33342},[355],[33344],{"type":155,"value":358},{"type":150,"tag":169,"props":33346,"children":33348},{"className":33347},[259],[33349],{"type":150,"tag":169,"props":33350,"children":33352},{"className":33351,"style":4728},[264],[33353],{"type":150,"tag":169,"props":33354,"children":33355},{},[],{"type":150,"tag":169,"props":33357,"children":33359},{"className":33358},[172],[33360],{"type":150,"tag":169,"props":33361,"children":33363},{"className":33362},[177],[33364,33438],{"type":150,"tag":169,"props":33365,"children":33367},{"className":33366},[182],[33368],{"type":150,"tag":185,"props":33369,"children":33370},{"xmlns":187,"display":188},[33371],{"type":150,"tag":191,"props":33372,"children":33373},{},[33374,33433],{"type":150,"tag":195,"props":33375,"children":33376},{},[33377,33388,33392,33396,33400,33411,33422],{"type":150,"tag":401,"props":33378,"children":33379},{},[33380,33384],{"type":150,"tag":405,"props":33381,"children":33382},{},[33383],{"type":155,"value":1208},{"type":150,"tag":405,"props":33385,"children":33386},{},[33387],{"type":155,"value":1213},{"type":150,"tag":417,"props":33389,"children":33390},{},[33391],{"type":155,"value":1045},{"type":150,"tag":405,"props":33393,"children":33394},{},[33395],{"type":155,"value":32984},{"type":150,"tag":417,"props":33397,"children":33398},{"stretchy":207},[33399],{"type":155,"value":972},{"type":150,"tag":401,"props":33401,"children":33402},{},[33403,33407],{"type":150,"tag":405,"props":33404,"children":33405},{},[33406],{"type":155,"value":1845},{"type":150,"tag":405,"props":33408,"children":33409},{},[33410],{"type":155,"value":1213},{"type":150,"tag":746,"props":33412,"children":33413},{},[33414,33418],{"type":150,"tag":417,"props":33415,"children":33416},{"stretchy":207},[33417],{"type":155,"value":982},{"type":150,"tag":405,"props":33419,"children":33420},{"mathvariant":1796},[33421],{"type":155,"value":2052},{"type":150,"tag":401,"props":33423,"children":33424},{},[33425,33429],{"type":150,"tag":405,"props":33426,"children":33427},{},[33428],{"type":155,"value":32945},{"type":150,"tag":405,"props":33430,"children":33431},{},[33432],{"type":155,"value":1213},{"type":150,"tag":223,"props":33434,"children":33435},{"encoding":225},[33436],{"type":155,"value":33437},"o_t = \\phi(q_t)^\\top S_t",{"type":150,"tag":169,"props":33439,"children":33441},{"className":33440,"ariaHidden":214},[232],[33442,33520],{"type":150,"tag":169,"props":33443,"children":33445},{"className":33444},[237],[33446,33450,33507,33511,33516],{"type":150,"tag":169,"props":33447,"children":33449},{"className":33448,"style":1343},[242],[],{"type":150,"tag":169,"props":33451,"children":33453},{"className":33452},[248],[33454,33459],{"type":150,"tag":169,"props":33455,"children":33457},{"className":33456},[248,489],[33458],{"type":155,"value":1208},{"type":150,"tag":169,"props":33460,"children":33462},{"className":33461},[495],[33463],{"type":150,"tag":169,"props":33464,"children":33466},{"className":33465},[253,254],[33467,33496],{"type":150,"tag":169,"props":33468,"children":33470},{"className":33469},[259],[33471,33491],{"type":150,"tag":169,"props":33472,"children":33474},{"className":33473,"style":1369},[264],[33475],{"type":150,"tag":169,"props":33476,"children":33477},{"style":512},[33478,33482],{"type":150,"tag":169,"props":33479,"children":33481},{"className":33480,"style":517},[274],[],{"type":150,"tag":169,"props":33483,"children":33485},{"className":33484},[522,523,524,525],[33486],{"type":150,"tag":169,"props":33487,"children":33489},{"className":33488},[248,489,525],[33490],{"type":155,"value":1213},{"type":150,"tag":169,"props":33492,"children":33494},{"className":33493},[355],[33495],{"type":155,"value":358},{"type":150,"tag":169,"props":33497,"children":33499},{"className":33498},[259],[33500],{"type":150,"tag":169,"props":33501,"children":33503},{"className":33502,"style":544},[264],[33504],{"type":150,"tag":169,"props":33505,"children":33506},{},[],{"type":150,"tag":169,"props":33508,"children":33510},{"className":33509,"style":839},[558],[],{"type":150,"tag":169,"props":33512,"children":33514},{"className":33513},[844],[33515],{"type":155,"value":1045},{"type":150,"tag":169,"props":33517,"children":33519},{"className":33518,"style":839},[558],[],{"type":150,"tag":169,"props":33521,"children":33523},{"className":33522},[237],[33524,33528,33533,33538,33595,33636],{"type":150,"tag":169,"props":33525,"children":33527},{"className":33526,"style":33208},[242],[],{"type":150,"tag":169,"props":33529,"children":33531},{"className":33530},[248,489],[33532],{"type":155,"value":32984},{"type":150,"tag":169,"props":33534,"children":33536},{"className":33535},[1004],[33537],{"type":155,"value":972},{"type":150,"tag":169,"props":33539,"children":33541},{"className":33540},[248],[33542,33547],{"type":150,"tag":169,"props":33543,"children":33545},{"className":33544,"style":2329},[248,489],[33546],{"type":155,"value":1845},{"type":150,"tag":169,"props":33548,"children":33550},{"className":33549},[495],[33551],{"type":150,"tag":169,"props":33552,"children":33554},{"className":33553},[253,254],[33555,33584],{"type":150,"tag":169,"props":33556,"children":33558},{"className":33557},[259],[33559,33579],{"type":150,"tag":169,"props":33560,"children":33562},{"className":33561,"style":1369},[264],[33563],{"type":150,"tag":169,"props":33564,"children":33565},{"style":14417},[33566,33570],{"type":150,"tag":169,"props":33567,"children":33569},{"className":33568,"style":517},[274],[],{"type":150,"tag":169,"props":33571,"children":33573},{"className":33572},[522,523,524,525],[33574],{"type":150,"tag":169,"props":33575,"children":33577},{"className":33576},[248,489,525],[33578],{"type":155,"value":1213},{"type":150,"tag":169,"props":33580,"children":33582},{"className":33581},[355],[33583],{"type":155,"value":358},{"type":150,"tag":169,"props":33585,"children":33587},{"className":33586},[259],[33588],{"type":150,"tag":169,"props":33589,"children":33591},{"className":33590,"style":544},[264],[33592],{"type":150,"tag":169,"props":33593,"children":33594},{},[],{"type":150,"tag":169,"props":33596,"children":33598},{"className":33597},[1015],[33599,33604],{"type":150,"tag":169,"props":33600,"children":33602},{"className":33601},[1015],[33603],{"type":155,"value":982},{"type":150,"tag":169,"props":33605,"children":33607},{"className":33606},[495],[33608],{"type":150,"tag":169,"props":33609,"children":33611},{"className":33610},[253],[33612],{"type":150,"tag":169,"props":33613,"children":33615},{"className":33614},[259],[33616],{"type":150,"tag":169,"props":33617,"children":33619},{"className":33618,"style":3999},[264],[33620],{"type":150,"tag":169,"props":33621,"children":33622},{"style":3908},[33623,33627],{"type":150,"tag":169,"props":33624,"children":33626},{"className":33625,"style":517},[274],[],{"type":150,"tag":169,"props":33628,"children":33630},{"className":33629},[522,523,524,525],[33631],{"type":150,"tag":169,"props":33632,"children":33634},{"className":33633},[248,525],[33635],{"type":155,"value":2052},{"type":150,"tag":169,"props":33637,"children":33639},{"className":33638},[248],[33640,33645],{"type":150,"tag":169,"props":33641,"children":33643},{"className":33642,"style":33043},[248,489],[33644],{"type":155,"value":32945},{"type":150,"tag":169,"props":33646,"children":33648},{"className":33647},[495],[33649],{"type":150,"tag":169,"props":33650,"children":33652},{"className":33651},[253,254],[33653,33682],{"type":150,"tag":169,"props":33654,"children":33656},{"className":33655},[259],[33657,33677],{"type":150,"tag":169,"props":33658,"children":33660},{"className":33659,"style":1369},[264],[33661],{"type":150,"tag":169,"props":33662,"children":33663},{"style":33064},[33664,33668],{"type":150,"tag":169,"props":33665,"children":33667},{"className":33666,"style":517},[274],[],{"type":150,"tag":169,"props":33669,"children":33671},{"className":33670},[522,523,524,525],[33672],{"type":150,"tag":169,"props":33673,"children":33675},{"className":33674},[248,489,525],[33676],{"type":155,"value":1213},{"type":150,"tag":169,"props":33678,"children":33680},{"className":33679},[355],[33681],{"type":155,"value":358},{"type":150,"tag":169,"props":33683,"children":33685},{"className":33684},[259],[33686],{"type":150,"tag":169,"props":33687,"children":33689},{"className":33688,"style":544},[264],[33690],{"type":150,"tag":169,"props":33691,"children":33692},{},[],{"type":150,"tag":151,"props":33694,"children":33695},{},[33696,33698,33767],{"type":155,"value":33697},"每步只更新一个 state，复杂度 ",{"type":150,"tag":169,"props":33699,"children":33701},{"className":33700},[177],[33702,33735],{"type":150,"tag":169,"props":33703,"children":33705},{"className":33704},[182],[33706],{"type":150,"tag":185,"props":33707,"children":33708},{"xmlns":187},[33709],{"type":150,"tag":191,"props":33710,"children":33711},{},[33712,33731],{"type":150,"tag":195,"props":33713,"children":33714},{},[33715,33719,33723,33727],{"type":150,"tag":405,"props":33716,"children":33717},{},[33718],{"type":155,"value":6923},{"type":150,"tag":417,"props":33720,"children":33721},{"stretchy":207},[33722],{"type":155,"value":972},{"type":150,"tag":405,"props":33724,"children":33725},{},[33726],{"type":155,"value":5591},{"type":150,"tag":417,"props":33728,"children":33729},{"stretchy":207},[33730],{"type":155,"value":982},{"type":150,"tag":223,"props":33732,"children":33733},{"encoding":225},[33734],{"type":155,"value":32183},{"type":150,"tag":169,"props":33736,"children":33738},{"className":33737,"ariaHidden":214},[232],[33739],{"type":150,"tag":169,"props":33740,"children":33742},{"className":33741},[237],[33743,33747,33752,33757,33762],{"type":150,"tag":169,"props":33744,"children":33746},{"className":33745,"style":999},[242],[],{"type":150,"tag":169,"props":33748,"children":33750},{"className":33749,"style":6968},[248,489],[33751],{"type":155,"value":6923},{"type":150,"tag":169,"props":33753,"children":33755},{"className":33754},[1004],[33756],{"type":155,"value":972},{"type":150,"tag":169,"props":33758,"children":33760},{"className":33759,"style":5780},[248,489],[33761],{"type":155,"value":5591},{"type":150,"tag":169,"props":33763,"children":33765},{"className":33764},[1015],[33766],{"type":155,"value":982},{"type":155,"value":33768},"。看到这个式子会觉得眼熟。它其实是：",{"type":150,"tag":169,"props":33770,"children":33772},{"className":33771},[172],[33773],{"type":150,"tag":169,"props":33774,"children":33776},{"className":33775},[177],[33777,33824],{"type":150,"tag":169,"props":33778,"children":33780},{"className":33779},[182],[33781],{"type":150,"tag":185,"props":33782,"children":33783},{"xmlns":187,"display":188},[33784],{"type":150,"tag":191,"props":33785,"children":33786},{},[33787,33819],{"type":150,"tag":195,"props":33788,"children":33789},{},[33790],{"type":150,"tag":199,"props":33791,"children":33792},{"notation":201},[33793],{"type":150,"tag":204,"props":33794,"children":33795},{"scriptlevel":206,"displaystyle":207},[33796],{"type":150,"tag":204,"props":33797,"children":33798},{"scriptlevel":206,"displaystyle":207},[33799],{"type":150,"tag":204,"props":33800,"children":33801},{"scriptlevel":206,"displaystyle":214},[33802],{"type":150,"tag":195,"props":33803,"children":33804},{},[33805,33810,33814],{"type":150,"tag":217,"props":33806,"children":33807},{},[33808],{"type":155,"value":33809},"LSTM 式的 recurrent state",{"type":150,"tag":417,"props":33811,"children":33812},{},[33813],{"type":155,"value":5136},{"type":150,"tag":217,"props":33815,"children":33816},{},[33817],{"type":155,"value":33818},"Attention 式的 query-key retrieval",{"type":150,"tag":223,"props":33820,"children":33821},{"encoding":225},[33822],{"type":155,"value":33823},"\\boxed{\\text{LSTM 式的 recurrent state} + \\text{Attention 式的 query-key retrieval}}",{"type":150,"tag":169,"props":33825,"children":33827},{"className":33826,"ariaHidden":214},[232],[33828],{"type":150,"tag":169,"props":33829,"children":33831},{"className":33830},[237],[33832,33836],{"type":150,"tag":169,"props":33833,"children":33835},{"className":33834,"style":30651},[242],[],{"type":150,"tag":169,"props":33837,"children":33839},{"className":33838},[248],[33840],{"type":150,"tag":169,"props":33841,"children":33843},{"className":33842},[253,254],[33844,33943],{"type":150,"tag":169,"props":33845,"children":33847},{"className":33846},[259],[33848,33938],{"type":150,"tag":169,"props":33849,"children":33851},{"className":33850,"style":30668},[264],[33852,33927],{"type":150,"tag":169,"props":33853,"children":33854},{"style":30672},[33855,33859],{"type":150,"tag":169,"props":33856,"children":33858},{"className":33857,"style":30677},[274],[],{"type":150,"tag":169,"props":33860,"children":33862},{"className":33861},[280],[33863],{"type":150,"tag":169,"props":33864,"children":33866},{"className":33865},[248],[33867],{"type":150,"tag":169,"props":33868,"children":33870},{"className":33869},[248],[33871,33893,33897,33902,33906],{"type":150,"tag":169,"props":33872,"children":33874},{"className":33873},[248,155],[33875,33881,33887],{"type":150,"tag":169,"props":33876,"children":33878},{"className":33877},[248],[33879],{"type":155,"value":33880},"LSTM ",{"type":150,"tag":169,"props":33882,"children":33884},{"className":33883},[248,297],[33885],{"type":155,"value":33886},"式的",{"type":150,"tag":169,"props":33888,"children":33890},{"className":33889},[248],[33891],{"type":155,"value":33892}," recurrent state",{"type":150,"tag":169,"props":33894,"children":33896},{"className":33895,"style":5167},[558],[],{"type":150,"tag":169,"props":33898,"children":33900},{"className":33899},[5172],[33901],{"type":155,"value":5136},{"type":150,"tag":169,"props":33903,"children":33905},{"className":33904,"style":5167},[558],[],{"type":150,"tag":169,"props":33907,"children":33909},{"className":33908},[248,155],[33910,33916,33921],{"type":150,"tag":169,"props":33911,"children":33913},{"className":33912},[248],[33914],{"type":155,"value":33915},"Attention ",{"type":150,"tag":169,"props":33917,"children":33919},{"className":33918},[248,297],[33920],{"type":155,"value":33886},{"type":150,"tag":169,"props":33922,"children":33924},{"className":33923},[248],[33925],{"type":155,"value":33926}," query-key retrieval",{"type":150,"tag":169,"props":33928,"children":33929},{"style":30721},[33930,33934],{"type":150,"tag":169,"props":33931,"children":33933},{"className":33932,"style":30677},[274],[],{"type":150,"tag":169,"props":33935,"children":33937},{"className":33936,"style":30730},[348,349],[],{"type":150,"tag":169,"props":33939,"children":33941},{"className":33940},[355],[33942],{"type":155,"value":358},{"type":150,"tag":169,"props":33944,"children":33946},{"className":33945},[259],[33947],{"type":150,"tag":169,"props":33948,"children":33950},{"className":33949,"style":366},[264],[33951],{"type":150,"tag":169,"props":33952,"children":33953},{},[],{"type":150,"tag":151,"props":33955,"children":33956},{},[33957,33959,33964,33966,34068,34070,34265,34267,34309],{"type":155,"value":33958},"所以它像一个\"现代 LSTM\"是有道理的，不是公式恰巧长得像。它重新选择了",{"type":150,"tag":158,"props":33960,"children":33961},{},[33962],{"type":155,"value":33963},"压缩历史，而不是保存历史",{"type":155,"value":33965},"。代价是信息碰撞。不同的 key 的信息全挤进同一个 state，容量有限，互相干扰。这件事值得展开一点，因为它是 Linear Attention 的核心缺陷，也是判断它能不能替代 full attention 的关键。state ",{"type":150,"tag":169,"props":33967,"children":33969},{"className":33968},[177],[33970,33999],{"type":150,"tag":169,"props":33971,"children":33973},{"className":33972},[182],[33974],{"type":150,"tag":185,"props":33975,"children":33976},{"xmlns":187},[33977],{"type":150,"tag":191,"props":33978,"children":33979},{},[33980,33994],{"type":150,"tag":195,"props":33981,"children":33982},{},[33983],{"type":150,"tag":401,"props":33984,"children":33985},{},[33986,33990],{"type":150,"tag":405,"props":33987,"children":33988},{},[33989],{"type":155,"value":32945},{"type":150,"tag":405,"props":33991,"children":33992},{},[33993],{"type":155,"value":1213},{"type":150,"tag":223,"props":33995,"children":33996},{"encoding":225},[33997],{"type":155,"value":33998},"S_t",{"type":150,"tag":169,"props":34000,"children":34002},{"className":34001,"ariaHidden":214},[232],[34003],{"type":150,"tag":169,"props":34004,"children":34006},{"className":34005},[237],[34007,34011],{"type":150,"tag":169,"props":34008,"children":34010},{"className":34009,"style":6364},[242],[],{"type":150,"tag":169,"props":34012,"children":34014},{"className":34013},[248],[34015,34020],{"type":150,"tag":169,"props":34016,"children":34018},{"className":34017,"style":33043},[248,489],[34019],{"type":155,"value":32945},{"type":150,"tag":169,"props":34021,"children":34023},{"className":34022},[495],[34024],{"type":150,"tag":169,"props":34025,"children":34027},{"className":34026},[253,254],[34028,34057],{"type":150,"tag":169,"props":34029,"children":34031},{"className":34030},[259],[34032,34052],{"type":150,"tag":169,"props":34033,"children":34035},{"className":34034,"style":1369},[264],[34036],{"type":150,"tag":169,"props":34037,"children":34038},{"style":33064},[34039,34043],{"type":150,"tag":169,"props":34040,"children":34042},{"className":34041,"style":517},[274],[],{"type":150,"tag":169,"props":34044,"children":34046},{"className":34045},[522,523,524,525],[34047],{"type":150,"tag":169,"props":34048,"children":34050},{"className":34049},[248,489,525],[34051],{"type":155,"value":1213},{"type":150,"tag":169,"props":34053,"children":34055},{"className":34054},[355],[34056],{"type":155,"value":358},{"type":150,"tag":169,"props":34058,"children":34060},{"className":34059},[259],[34061],{"type":150,"tag":169,"props":34062,"children":34064},{"className":34063,"style":544},[264],[34065],{"type":150,"tag":169,"props":34066,"children":34067},{},[],{"type":155,"value":34069}," 的大小是固定的，跟 ",{"type":150,"tag":169,"props":34071,"children":34073},{"className":34072},[177],[34074,34118],{"type":150,"tag":169,"props":34075,"children":34077},{"className":34076},[182],[34078],{"type":150,"tag":185,"props":34079,"children":34080},{"xmlns":187},[34081],{"type":150,"tag":191,"props":34082,"children":34083},{},[34084,34113],{"type":150,"tag":195,"props":34085,"children":34086},{},[34087,34098,34102],{"type":150,"tag":401,"props":34088,"children":34089},{},[34090,34094],{"type":150,"tag":405,"props":34091,"children":34092},{},[34093],{"type":155,"value":759},{"type":150,"tag":405,"props":34095,"children":34096},{},[34097],{"type":155,"value":1876},{"type":150,"tag":417,"props":34099,"children":34100},{},[34101],{"type":155,"value":5554},{"type":150,"tag":401,"props":34103,"children":34104},{},[34105,34109],{"type":150,"tag":405,"props":34106,"children":34107},{},[34108],{"type":155,"value":759},{"type":150,"tag":405,"props":34110,"children":34111},{},[34112],{"type":155,"value":1915},{"type":150,"tag":223,"props":34114,"children":34115},{"encoding":225},[34116],{"type":155,"value":34117},"d_k \\times d_v",{"type":150,"tag":169,"props":34119,"children":34121},{"className":34120,"ariaHidden":214},[232],[34122,34200],{"type":150,"tag":169,"props":34123,"children":34125},{"className":34124},[237],[34126,34130,34187,34191,34196],{"type":150,"tag":169,"props":34127,"children":34129},{"className":34128,"style":4935},[242],[],{"type":150,"tag":169,"props":34131,"children":34133},{"className":34132},[248],[34134,34139],{"type":150,"tag":169,"props":34135,"children":34137},{"className":34136},[248,489],[34138],{"type":155,"value":759},{"type":150,"tag":169,"props":34140,"children":34142},{"className":34141},[495],[34143],{"type":150,"tag":169,"props":34144,"children":34146},{"className":34145},[253,254],[34147,34176],{"type":150,"tag":169,"props":34148,"children":34150},{"className":34149},[259],[34151,34171],{"type":150,"tag":169,"props":34152,"children":34154},{"className":34153,"style":674},[264],[34155],{"type":150,"tag":169,"props":34156,"children":34157},{"style":512},[34158,34162],{"type":150,"tag":169,"props":34159,"children":34161},{"className":34160,"style":517},[274],[],{"type":150,"tag":169,"props":34163,"children":34165},{"className":34164},[522,523,524,525],[34166],{"type":150,"tag":169,"props":34167,"children":34169},{"className":34168,"style":2427},[248,489,525],[34170],{"type":155,"value":1876},{"type":150,"tag":169,"props":34172,"children":34174},{"className":34173},[355],[34175],{"type":155,"value":358},{"type":150,"tag":169,"props":34177,"children":34179},{"className":34178},[259],[34180],{"type":150,"tag":169,"props":34181,"children":34183},{"className":34182,"style":544},[264],[34184],{"type":150,"tag":169,"props":34185,"children":34186},{},[],{"type":150,"tag":169,"props":34188,"children":34190},{"className":34189,"style":5167},[558],[],{"type":150,"tag":169,"props":34192,"children":34194},{"className":34193},[5172],[34195],{"type":155,"value":5554},{"type":150,"tag":169,"props":34197,"children":34199},{"className":34198,"style":5167},[558],[],{"type":150,"tag":169,"props":34201,"children":34203},{"className":34202},[237],[34204,34208],{"type":150,"tag":169,"props":34205,"children":34207},{"className":34206,"style":4935},[242],[],{"type":150,"tag":169,"props":34209,"children":34211},{"className":34210},[248],[34212,34217],{"type":150,"tag":169,"props":34213,"children":34215},{"className":34214},[248,489],[34216],{"type":155,"value":759},{"type":150,"tag":169,"props":34218,"children":34220},{"className":34219},[495],[34221],{"type":150,"tag":169,"props":34222,"children":34224},{"className":34223},[253,254],[34225,34254],{"type":150,"tag":169,"props":34226,"children":34228},{"className":34227},[259],[34229,34249],{"type":150,"tag":169,"props":34230,"children":34232},{"className":34231,"style":5039},[264],[34233],{"type":150,"tag":169,"props":34234,"children":34235},{"style":512},[34236,34240],{"type":150,"tag":169,"props":34237,"children":34239},{"className":34238,"style":517},[274],[],{"type":150,"tag":169,"props":34241,"children":34243},{"className":34242},[522,523,524,525],[34244],{"type":150,"tag":169,"props":34245,"children":34247},{"className":34246,"style":2329},[248,489,525],[34248],{"type":155,"value":1915},{"type":150,"tag":169,"props":34250,"children":34252},{"className":34251},[355],[34253],{"type":155,"value":358},{"type":150,"tag":169,"props":34255,"children":34257},{"className":34256},[259],[34258],{"type":150,"tag":169,"props":34259,"children":34261},{"className":34260,"style":544},[264],[34262],{"type":150,"tag":169,"props":34263,"children":34264},{},[],{"type":155,"value":34266}," 同阶，不随 ",{"type":150,"tag":169,"props":34268,"children":34270},{"className":34269},[177],[34271,34292],{"type":150,"tag":169,"props":34272,"children":34274},{"className":34273},[182],[34275],{"type":150,"tag":185,"props":34276,"children":34277},{"xmlns":187},[34278],{"type":150,"tag":191,"props":34279,"children":34280},{},[34281,34288],{"type":150,"tag":195,"props":34282,"children":34283},{},[34284],{"type":150,"tag":405,"props":34285,"children":34286},{},[34287],{"type":155,"value":5591},{"type":150,"tag":223,"props":34289,"children":34290},{"encoding":225},[34291],{"type":155,"value":5591},{"type":150,"tag":169,"props":34293,"children":34295},{"className":34294,"ariaHidden":214},[232],[34296],{"type":150,"tag":169,"props":34297,"children":34299},{"className":34298},[237],[34300,34304],{"type":150,"tag":169,"props":34301,"children":34303},{"className":34302,"style":5854},[242],[],{"type":150,"tag":169,"props":34305,"children":34307},{"className":34306,"style":5780},[248,489],[34308],{"type":155,"value":5591},{"type":155,"value":34310}," 增长。",{"type":150,"tag":151,"props":34312,"children":34313},{},[34314],{"type":155,"value":34315},"而它要装的是全部历史的信息。序列越长，需要塞进去的东西越多，碰撞就越严重。",{"type":150,"tag":151,"props":34317,"children":34318},{},[34319],{"type":155,"value":34320},"具体表现是：想精确取回某一个早先 token 的信息时，取回来的其实是很多 token 叠加后的结果。full attention 能精确地只挑那一个，Linear Attention 做不到。所以它的失效方式不是\"忘了\"，而是\"记混了\"。这两个词听起来差不多，实际差很远。前者可以通过更好的门控缓解，后者是容量上限本身决定的。",{"type":150,"tag":151,"props":34322,"children":34323},{},[34324],{"type":150,"tag":158,"props":34325,"children":34326},{},[34327],{"type":155,"value":34328},"Mamba \u002F SSM 是什么？",{"type":150,"tag":151,"props":34330,"children":34331},{},[34332,34334,34447,34449,34518],{"type":155,"value":34333},"SSM（State Space Model）是这条线的另一个分支，典型代表是 Mamba。它和 Linear Attention 的数学形式不同，但目标一样：用固定大小的状态代替完整历史，把 ",{"type":150,"tag":169,"props":34335,"children":34337},{"className":34336},[177],[34338,34379],{"type":150,"tag":169,"props":34339,"children":34341},{"className":34340},[182],[34342],{"type":150,"tag":185,"props":34343,"children":34344},{"xmlns":187},[34345],{"type":150,"tag":191,"props":34346,"children":34347},{},[34348,34374],{"type":150,"tag":195,"props":34349,"children":34350},{},[34351,34355,34359,34370],{"type":150,"tag":405,"props":34352,"children":34353},{},[34354],{"type":155,"value":6923},{"type":150,"tag":417,"props":34356,"children":34357},{"stretchy":207},[34358],{"type":155,"value":972},{"type":150,"tag":746,"props":34360,"children":34361},{},[34362,34366],{"type":150,"tag":405,"props":34363,"children":34364},{},[34365],{"type":155,"value":5591},{"type":150,"tag":411,"props":34367,"children":34368},{},[34369],{"type":155,"value":433},{"type":150,"tag":417,"props":34371,"children":34372},{"stretchy":207},[34373],{"type":155,"value":982},{"type":150,"tag":223,"props":34375,"children":34376},{"encoding":225},[34377],{"type":155,"value":34378},"O(N^2)",{"type":150,"tag":169,"props":34380,"children":34382},{"className":34381,"ariaHidden":214},[232],[34383],{"type":150,"tag":169,"props":34384,"children":34386},{"className":34385},[237],[34387,34391,34396,34401,34442],{"type":150,"tag":169,"props":34388,"children":34390},{"className":34389,"style":6963},[242],[],{"type":150,"tag":169,"props":34392,"children":34394},{"className":34393,"style":6968},[248,489],[34395],{"type":155,"value":6923},{"type":150,"tag":169,"props":34397,"children":34399},{"className":34398},[1004],[34400],{"type":155,"value":972},{"type":150,"tag":169,"props":34402,"children":34404},{"className":34403},[248],[34405,34410],{"type":150,"tag":169,"props":34406,"children":34408},{"className":34407,"style":5780},[248,489],[34409],{"type":155,"value":5591},{"type":150,"tag":169,"props":34411,"children":34413},{"className":34412},[495],[34414],{"type":150,"tag":169,"props":34415,"children":34417},{"className":34416},[253],[34418],{"type":150,"tag":169,"props":34419,"children":34421},{"className":34420},[259],[34422],{"type":150,"tag":169,"props":34423,"children":34425},{"className":34424,"style":7005},[264],[34426],{"type":150,"tag":169,"props":34427,"children":34428},{"style":888},[34429,34433],{"type":150,"tag":169,"props":34430,"children":34432},{"className":34431,"style":517},[274],[],{"type":150,"tag":169,"props":34434,"children":34436},{"className":34435},[522,523,524,525],[34437],{"type":150,"tag":169,"props":34438,"children":34440},{"className":34439},[248,525],[34441],{"type":155,"value":433},{"type":150,"tag":169,"props":34443,"children":34445},{"className":34444},[1015],[34446],{"type":155,"value":982},{"type":155,"value":34448}," 降到 ",{"type":150,"tag":169,"props":34450,"children":34452},{"className":34451},[177],[34453,34486],{"type":150,"tag":169,"props":34454,"children":34456},{"className":34455},[182],[34457],{"type":150,"tag":185,"props":34458,"children":34459},{"xmlns":187},[34460],{"type":150,"tag":191,"props":34461,"children":34462},{},[34463,34482],{"type":150,"tag":195,"props":34464,"children":34465},{},[34466,34470,34474,34478],{"type":150,"tag":405,"props":34467,"children":34468},{},[34469],{"type":155,"value":6923},{"type":150,"tag":417,"props":34471,"children":34472},{"stretchy":207},[34473],{"type":155,"value":972},{"type":150,"tag":405,"props":34475,"children":34476},{},[34477],{"type":155,"value":5591},{"type":150,"tag":417,"props":34479,"children":34480},{"stretchy":207},[34481],{"type":155,"value":982},{"type":150,"tag":223,"props":34483,"children":34484},{"encoding":225},[34485],{"type":155,"value":32183},{"type":150,"tag":169,"props":34487,"children":34489},{"className":34488,"ariaHidden":214},[232],[34490],{"type":150,"tag":169,"props":34491,"children":34493},{"className":34492},[237],[34494,34498,34503,34508,34513],{"type":150,"tag":169,"props":34495,"children":34497},{"className":34496,"style":999},[242],[],{"type":150,"tag":169,"props":34499,"children":34501},{"className":34500,"style":6968},[248,489],[34502],{"type":155,"value":6923},{"type":150,"tag":169,"props":34504,"children":34506},{"className":34505},[1004],[34507],{"type":155,"value":972},{"type":150,"tag":169,"props":34509,"children":34511},{"className":34510,"style":5780},[248,489],[34512],{"type":155,"value":5591},{"type":150,"tag":169,"props":34514,"children":34516},{"className":34515},[1015],[34517],{"type":155,"value":982},{"type":155,"value":34519},"。Mamba 的特点在状态更新的方式上。它让状态转移矩阵随输入变化，相当于给状态加了一层输入相关的门控。这和 LSTM 的门控思路一脉相承，只是换了个数学外衣。",{"type":150,"tag":151,"props":34521,"children":34522},{},[34523,34525,34530],{"type":155,"value":34524},"所以往上看，Mamba、RWKV、RetNet、DeltaNet、Gated DeltaNet、Kimi Linear，看起来像一堆论文名字动物园，其实都在回答同一个问题：",{"type":150,"tag":158,"props":34526,"children":34527},{},[34528],{"type":155,"value":34529},"怎么用固定大小的记忆，尽可能少地丢掉历史。",{"type":155,"value":34531}," 于是有了 Hybrid：大部分层用廉价的 state memory，少数几层保留对完整历史的随机访问。",{"type":150,"tag":169,"props":34533,"children":34535},{"className":34534},[172],[34536],{"type":150,"tag":169,"props":34537,"children":34539},{"className":34538},[177],[34540,34572],{"type":150,"tag":169,"props":34541,"children":34543},{"className":34542},[182],[34544],{"type":150,"tag":185,"props":34545,"children":34546},{"xmlns":187,"display":188},[34547],{"type":150,"tag":191,"props":34548,"children":34549},{},[34550,34567],{"type":150,"tag":195,"props":34551,"children":34552},{},[34553,34558,34562],{"type":150,"tag":217,"props":34554,"children":34555},{},[34556],{"type":155,"value":34557},"高速压缩记忆",{"type":150,"tag":417,"props":34559,"children":34560},{},[34561],{"type":155,"value":5136},{"type":150,"tag":217,"props":34563,"children":34564},{},[34565],{"type":155,"value":34566},"偶尔随机访问原始信息",{"type":150,"tag":223,"props":34568,"children":34569},{"encoding":225},[34570],{"type":155,"value":34571},"\\text{高速压缩记忆} + \\text{偶尔随机访问原始信息}",{"type":150,"tag":169,"props":34573,"children":34575},{"className":34574,"ariaHidden":214},[232],[34576,34606],{"type":150,"tag":169,"props":34577,"children":34579},{"className":34578},[237],[34580,34584,34593,34597,34602],{"type":150,"tag":169,"props":34581,"children":34583},{"className":34582,"style":5644},[242],[],{"type":150,"tag":169,"props":34585,"children":34587},{"className":34586},[248,155],[34588],{"type":150,"tag":169,"props":34589,"children":34591},{"className":34590},[248,297],[34592],{"type":155,"value":34557},{"type":150,"tag":169,"props":34594,"children":34596},{"className":34595,"style":5167},[558],[],{"type":150,"tag":169,"props":34598,"children":34600},{"className":34599},[5172],[34601],{"type":155,"value":5136},{"type":150,"tag":169,"props":34603,"children":34605},{"className":34604,"style":5167},[558],[],{"type":150,"tag":169,"props":34607,"children":34609},{"className":34608},[237],[34610,34614],{"type":150,"tag":169,"props":34611,"children":34613},{"className":34612,"style":5854},[242],[],{"type":150,"tag":169,"props":34615,"children":34617},{"className":34616},[248,155],[34618],{"type":150,"tag":169,"props":34619,"children":34621},{"className":34620},[248,297],[34622],{"type":155,"value":34566},{"type":150,"tag":151,"props":34624,"children":34625},{},[34626],{"type":155,"value":34627},"有个很漂亮的观察：这不是\"RNN 又赢了\"，而是大家终于发现 attention 的随机访问能力很好、RNN 的固定状态成本也很好，真正有意思的问题是怎么把两者揉到一起，而不是宗教战争式地选一个。",{"type":150,"tag":372,"props":34629,"children":34631},{"id":34630},"kimi-与-deepseek",[34632],{"type":155,"value":34633},"Kimi 与 DeepSeek",{"type":150,"tag":151,"props":34635,"children":34636},{},[34637],{"type":155,"value":34638},"到这里可以做一个全文最重要的比较。Kimi 和 DeepSeek 都在压历史，但压完之后的处理方式不一样：",{"type":150,"tag":169,"props":34640,"children":34642},{"className":34641},[172],[34643],{"type":150,"tag":169,"props":34644,"children":34646},{"className":34645},[177],[34647,34670],{"type":150,"tag":169,"props":34648,"children":34650},{"className":34649},[182],[34651],{"type":150,"tag":185,"props":34652,"children":34653},{"xmlns":187,"display":188},[34654],{"type":150,"tag":191,"props":34655,"children":34656},{},[34657,34665],{"type":150,"tag":195,"props":34658,"children":34659},{},[34660],{"type":150,"tag":217,"props":34661,"children":34662},{},[34663],{"type":155,"value":34664},"Kimi：compress history into state",{"type":150,"tag":223,"props":34666,"children":34667},{"encoding":225},[34668],{"type":155,"value":34669},"\\text{Kimi：compress history into state}",{"type":150,"tag":169,"props":34671,"children":34673},{"className":34672,"ariaHidden":214},[232],[34674],{"type":150,"tag":169,"props":34675,"children":34677},{"className":34676},[237],[34678,34682],{"type":150,"tag":169,"props":34679,"children":34681},{"className":34680,"style":1121},[242],[],{"type":150,"tag":169,"props":34683,"children":34685},{"className":34684},[248,155],[34686,34692,34697],{"type":150,"tag":169,"props":34687,"children":34689},{"className":34688},[248],[34690],{"type":155,"value":34691},"Kimi",{"type":150,"tag":169,"props":34693,"children":34695},{"className":34694},[248,297],[34696],{"type":155,"value":11339},{"type":150,"tag":169,"props":34698,"children":34700},{"className":34699},[248],[34701],{"type":155,"value":34702},"compress history into state",{"type":150,"tag":169,"props":34704,"children":34706},{"className":34705},[172],[34707],{"type":150,"tag":169,"props":34708,"children":34710},{"className":34709},[177],[34711,34734],{"type":150,"tag":169,"props":34712,"children":34714},{"className":34713},[182],[34715],{"type":150,"tag":185,"props":34716,"children":34717},{"xmlns":187,"display":188},[34718],{"type":150,"tag":191,"props":34719,"children":34720},{},[34721,34729],{"type":150,"tag":195,"props":34722,"children":34723},{},[34724],{"type":150,"tag":217,"props":34725,"children":34726},{},[34727],{"type":155,"value":34728},"DeepSeek：compress history but keep it retrievable",{"type":150,"tag":223,"props":34730,"children":34731},{"encoding":225},[34732],{"type":155,"value":34733},"\\text{DeepSeek：compress history but keep it retrievable}",{"type":150,"tag":169,"props":34735,"children":34737},{"className":34736,"ariaHidden":214},[232],[34738],{"type":150,"tag":169,"props":34739,"children":34741},{"className":34740},[237],[34742,34746],{"type":150,"tag":169,"props":34743,"children":34745},{"className":34744,"style":1121},[242],[],{"type":150,"tag":169,"props":34747,"children":34749},{"className":34748},[248,155],[34750,34755,34760],{"type":150,"tag":169,"props":34751,"children":34753},{"className":34752},[248],[34754],{"type":155,"value":14},{"type":150,"tag":169,"props":34756,"children":34758},{"className":34757},[248,297],[34759],{"type":155,"value":11339},{"type":150,"tag":169,"props":34761,"children":34763},{"className":34762},[248],[34764],{"type":155,"value":34765},"compress history but keep it retrievable",{"type":150,"tag":30755,"props":34767,"children":34769},{"id":34768},"state-machine-与-indexed-memory",[34770],{"type":155,"value":34771},"State Machine 与 Indexed Memory",{"type":150,"tag":151,"props":34773,"children":34774},{},[34775],{"type":155,"value":34776},"Kimi 的 Linear Attention 路线（KDA 那一支）是把历史递推成一个 state，查询时直接对 state 做操作：",{"type":150,"tag":34778,"props":34779,"children":34782},"pre",{"className":34780,"code":34781,"language":155,"meta":28,"style":28},"language-text shiki shiki-themes github-light github-dark","历史\n ↓\nrecurrent state\n ↓\nquery state\n",[34783],{"type":150,"tag":34784,"props":34785,"children":34786},"code",{"__ignoreMap":28},[34787,34797,34806,34815,34823],{"type":150,"tag":169,"props":34788,"children":34791},{"class":34789,"line":34790},"line",1,[34792],{"type":150,"tag":169,"props":34793,"children":34794},{},[34795],{"type":155,"value":34796},"历史\n",{"type":150,"tag":169,"props":34798,"children":34800},{"class":34789,"line":34799},2,[34801],{"type":150,"tag":169,"props":34802,"children":34803},{},[34804],{"type":155,"value":34805}," ↓\n",{"type":150,"tag":169,"props":34807,"children":34809},{"class":34789,"line":34808},3,[34810],{"type":150,"tag":169,"props":34811,"children":34812},{},[34813],{"type":155,"value":34814},"recurrent state\n",{"type":150,"tag":169,"props":34816,"children":34818},{"class":34789,"line":34817},4,[34819],{"type":150,"tag":169,"props":34820,"children":34821},{},[34822],{"type":155,"value":34805},{"type":150,"tag":169,"props":34824,"children":34826},{"class":34789,"line":34825},5,[34827],{"type":150,"tag":169,"props":34828,"children":34829},{},[34830],{"type":155,"value":34831},"query state\n",{"type":150,"tag":151,"props":34833,"children":34834},{},[34835],{"type":155,"value":34836},"DeepSeek 的路线是压缩，但压缩后的东西仍然带索引、可以被检索回来：",{"type":150,"tag":34778,"props":34838,"children":34840},{"className":34780,"code":34839,"language":155,"meta":28,"style":28},"历史\n ↓\ncompressed memory\n ↓\nindex \u002F retrieval\n ↓\nsparse attention\n",[34841],{"type":150,"tag":34784,"props":34842,"children":34843},{"__ignoreMap":28},[34844,34851,34858,34866,34873,34881,34889],{"type":150,"tag":169,"props":34845,"children":34846},{"class":34789,"line":34790},[34847],{"type":150,"tag":169,"props":34848,"children":34849},{},[34850],{"type":155,"value":34796},{"type":150,"tag":169,"props":34852,"children":34853},{"class":34789,"line":34799},[34854],{"type":150,"tag":169,"props":34855,"children":34856},{},[34857],{"type":155,"value":34805},{"type":150,"tag":169,"props":34859,"children":34860},{"class":34789,"line":34808},[34861],{"type":150,"tag":169,"props":34862,"children":34863},{},[34864],{"type":155,"value":34865},"compressed memory\n",{"type":150,"tag":169,"props":34867,"children":34868},{"class":34789,"line":34817},[34869],{"type":150,"tag":169,"props":34870,"children":34871},{},[34872],{"type":155,"value":34805},{"type":150,"tag":169,"props":34874,"children":34875},{"class":34789,"line":34825},[34876],{"type":150,"tag":169,"props":34877,"children":34878},{},[34879],{"type":155,"value":34880},"index \u002F retrieval\n",{"type":150,"tag":169,"props":34882,"children":34884},{"class":34789,"line":34883},6,[34885],{"type":150,"tag":169,"props":34886,"children":34887},{},[34888],{"type":155,"value":34805},{"type":150,"tag":169,"props":34890,"children":34892},{"class":34789,"line":34891},7,[34893],{"type":150,"tag":169,"props":34894,"children":34895},{},[34896],{"type":155,"value":34897},"sparse attention\n",{"type":150,"tag":151,"props":34899,"children":34900},{},[34901],{"type":155,"value":34902},"我把它们概括成：",{"type":150,"tag":169,"props":34904,"children":34906},{"className":34905},[172],[34907],{"type":150,"tag":169,"props":34908,"children":34910},{"className":34909},[177],[34911,34974],{"type":150,"tag":169,"props":34912,"children":34914},{"className":34913},[182],[34915],{"type":150,"tag":185,"props":34916,"children":34917},{"xmlns":187,"display":188},[34918],{"type":150,"tag":191,"props":34919,"children":34920},{},[34921,34969],{"type":150,"tag":195,"props":34922,"children":34923},{},[34924,34941,34944,34949,34952],{"type":150,"tag":199,"props":34925,"children":34926},{"notation":201},[34927],{"type":150,"tag":204,"props":34928,"children":34929},{"scriptlevel":206,"displaystyle":207},[34930],{"type":150,"tag":204,"props":34931,"children":34932},{"scriptlevel":206,"displaystyle":207},[34933],{"type":150,"tag":204,"props":34934,"children":34935},{"scriptlevel":206,"displaystyle":214},[34936],{"type":150,"tag":217,"props":34937,"children":34938},{},[34939],{"type":155,"value":34940},"State Machine",{"type":150,"tag":558,"props":34942,"children":34943},{"width":3491},[],{"type":150,"tag":217,"props":34945,"children":34946},{},[34947],{"type":155,"value":34948},"vs.",{"type":150,"tag":558,"props":34950,"children":34951},{"width":3491},[],{"type":150,"tag":199,"props":34953,"children":34954},{"notation":201},[34955],{"type":150,"tag":204,"props":34956,"children":34957},{"scriptlevel":206,"displaystyle":207},[34958],{"type":150,"tag":204,"props":34959,"children":34960},{"scriptlevel":206,"displaystyle":207},[34961],{"type":150,"tag":204,"props":34962,"children":34963},{"scriptlevel":206,"displaystyle":214},[34964],{"type":150,"tag":217,"props":34965,"children":34966},{},[34967],{"type":155,"value":34968},"Indexed Memory",{"type":150,"tag":223,"props":34970,"children":34971},{"encoding":225},[34972],{"type":155,"value":34973},"\\boxed{\\text{State Machine}} \\quad \\text{vs.} \\quad \\boxed{\\text{Indexed Memory}}",{"type":150,"tag":169,"props":34975,"children":34977},{"className":34976,"ariaHidden":214},[232],[34978],{"type":150,"tag":169,"props":34979,"children":34981},{"className":34980},[237],[34982,34986,35061,35065,35074,35078],{"type":150,"tag":169,"props":34983,"children":34985},{"className":34984,"style":30651},[242],[],{"type":150,"tag":169,"props":34987,"children":34989},{"className":34988},[248],[34990],{"type":150,"tag":169,"props":34991,"children":34993},{"className":34992},[253,254],[34994,35049],{"type":150,"tag":169,"props":34995,"children":34997},{"className":34996},[259],[34998,35044],{"type":150,"tag":169,"props":34999,"children":35001},{"className":35000,"style":30668},[264],[35002,35032],{"type":150,"tag":169,"props":35003,"children":35005},{"style":35004},"top:-3.3744em;",[35006,35011],{"type":150,"tag":169,"props":35007,"children":35010},{"className":35008,"style":35009},[274],"height:3.3744em;",[],{"type":150,"tag":169,"props":35012,"children":35014},{"className":35013},[280],[35015],{"type":150,"tag":169,"props":35016,"children":35018},{"className":35017},[248],[35019],{"type":150,"tag":169,"props":35020,"children":35022},{"className":35021},[248],[35023],{"type":150,"tag":169,"props":35024,"children":35026},{"className":35025},[248,155],[35027],{"type":150,"tag":169,"props":35028,"children":35030},{"className":35029},[248],[35031],{"type":155,"value":34940},{"type":150,"tag":169,"props":35033,"children":35034},{"style":30721},[35035,35039],{"type":150,"tag":169,"props":35036,"children":35038},{"className":35037,"style":35009},[274],[],{"type":150,"tag":169,"props":35040,"children":35043},{"className":35041,"style":35042},[348,349],"height:1.3744em;border-style:solid;border-width:0.04em;",[],{"type":150,"tag":169,"props":35045,"children":35047},{"className":35046},[355],[35048],{"type":155,"value":358},{"type":150,"tag":169,"props":35050,"children":35052},{"className":35051},[259],[35053],{"type":150,"tag":169,"props":35054,"children":35057},{"className":35055,"style":35056},[264],"height:0.34em;",[35058],{"type":150,"tag":169,"props":35059,"children":35060},{},[],{"type":150,"tag":169,"props":35062,"children":35064},{"className":35063,"style":4087},[558],[],{"type":150,"tag":169,"props":35066,"children":35068},{"className":35067},[248,155],[35069],{"type":150,"tag":169,"props":35070,"children":35072},{"className":35071},[248],[35073],{"type":155,"value":34948},{"type":150,"tag":169,"props":35075,"children":35077},{"className":35076,"style":4087},[558],[],{"type":150,"tag":169,"props":35079,"children":35081},{"className":35080},[248],[35082],{"type":150,"tag":169,"props":35083,"children":35085},{"className":35084},[253,254],[35086,35138],{"type":150,"tag":169,"props":35087,"children":35089},{"className":35088},[259],[35090,35133],{"type":150,"tag":169,"props":35091,"children":35093},{"className":35092,"style":30668},[264],[35094,35122],{"type":150,"tag":169,"props":35095,"children":35096},{"style":30672},[35097,35101],{"type":150,"tag":169,"props":35098,"children":35100},{"className":35099,"style":30677},[274],[],{"type":150,"tag":169,"props":35102,"children":35104},{"className":35103},[280],[35105],{"type":150,"tag":169,"props":35106,"children":35108},{"className":35107},[248],[35109],{"type":150,"tag":169,"props":35110,"children":35112},{"className":35111},[248],[35113],{"type":150,"tag":169,"props":35114,"children":35116},{"className":35115},[248,155],[35117],{"type":150,"tag":169,"props":35118,"children":35120},{"className":35119},[248],[35121],{"type":155,"value":34968},{"type":150,"tag":169,"props":35123,"children":35124},{"style":30721},[35125,35129],{"type":150,"tag":169,"props":35126,"children":35128},{"className":35127,"style":30677},[274],[],{"type":150,"tag":169,"props":35130,"children":35132},{"className":35131,"style":30730},[348,349],[],{"type":150,"tag":169,"props":35134,"children":35136},{"className":35135},[355],[35137],{"type":155,"value":358},{"type":150,"tag":169,"props":35139,"children":35141},{"className":35140},[259],[35142],{"type":150,"tag":169,"props":35143,"children":35145},{"className":35144,"style":366},[264],[35146],{"type":150,"tag":169,"props":35147,"children":35148},{},[],{"type":150,"tag":151,"props":35150,"children":35151},{},[35152,35154,35159,35161,35166],{"type":155,"value":35153},"这比说\"Linear Attention vs Sparse Attention\"有解释力。因为后者的区别看起来只是稀疏程度不同，而前者是两种根本不同的 memory 哲学：一个是",{"type":150,"tag":158,"props":35155,"children":35156},{},[35157],{"type":155,"value":35158},"把历史变成状态",{"type":155,"value":35160},"，一个是",{"type":150,"tag":158,"props":35162,"children":35163},{},[35164],{"type":155,"value":35165},"把历史变成可检索的索引",{"type":155,"value":35167},"。有一个细节能说明 Kimi 这边并不只是\"换了个 attention\"。",{"type":150,"tag":151,"props":35169,"children":35170},{},[35171],{"type":155,"value":35172},"K3 的技术报告把 refined training and data recipes、general\u002Fagentic\u002Fcoding RL、million-token agentic RL infrastructure、sandbox state，和 KDA、AttnRes、LatentMoE 并列成核心进展。注意这个列表的构成：一半是架构，一半是数据和基础设施。",{"type":150,"tag":151,"props":35174,"children":35175},{},[35176],{"type":155,"value":35177},"如果 KDA 真是能力跃迁的唯一来源，报告没必要把 RL infra 抬到同样的位置。",{"type":150,"tag":151,"props":35179,"children":35180},{},[35181,35183,35239,35241,35246,35248,35253],{"type":155,"value":35182},"反过来看，这也解释了 KDA 这类工作为什么重要。它让 ",{"type":150,"tag":169,"props":35184,"children":35186},{"className":35185},[177],[35187,35213],{"type":150,"tag":169,"props":35188,"children":35190},{"className":35189},[182],[35191],{"type":150,"tag":185,"props":35192,"children":35193},{"xmlns":187},[35194],{"type":150,"tag":191,"props":35195,"children":35196},{},[35197,35208],{"type":150,"tag":195,"props":35198,"children":35199},{},[35200,35204],{"type":150,"tag":411,"props":35201,"children":35202},{},[35203],{"type":155,"value":415},{"type":150,"tag":217,"props":35205,"children":35206},{},[35207],{"type":155,"value":6060},{"type":150,"tag":223,"props":35209,"children":35210},{"encoding":225},[35211],{"type":155,"value":35212},"1\\text{M}",{"type":150,"tag":169,"props":35214,"children":35216},{"className":35215,"ariaHidden":214},[232],[35217],{"type":150,"tag":169,"props":35218,"children":35220},{"className":35219},[237],[35221,35225,35230],{"type":150,"tag":169,"props":35222,"children":35224},{"className":35223,"style":5854},[242],[],{"type":150,"tag":169,"props":35226,"children":35228},{"className":35227},[248],[35229],{"type":155,"value":415},{"type":150,"tag":169,"props":35231,"children":35233},{"className":35232},[248,155],[35234],{"type":150,"tag":169,"props":35235,"children":35237},{"className":35236},[248],[35238],{"type":155,"value":6060},{"type":155,"value":35240}," context 和 long-horizon rollout 在计算上变得现实，于是以前喂不起的 RL trajectories 变得喂得起。架构在这里是",{"type":150,"tag":158,"props":35242,"children":35243},{},[35244],{"type":155,"value":35245},"开路的",{"type":155,"value":35247},"，不是",{"type":150,"tag":158,"props":35249,"children":35250},{},[35251],{"type":155,"value":35252},"发力的",{"type":155,"value":35254},"。DeepSeek 这边也一样。CSA\u002FHCA 最漂亮的地方未必是\"稀疏 attention 本身理解能力比 MLA 强\"，而是 1M context 变得足够便宜。",{"type":150,"tag":151,"props":35256,"children":35257},{},[35258,35260,35329,35331,35410],{"type":155,"value":35259},"那这两条路线谁能赢？我觉得决胜点不在复杂度写 ",{"type":150,"tag":169,"props":35261,"children":35263},{"className":35262},[177],[35264,35297],{"type":150,"tag":169,"props":35265,"children":35267},{"className":35266},[182],[35268],{"type":150,"tag":185,"props":35269,"children":35270},{"xmlns":187},[35271],{"type":150,"tag":191,"props":35272,"children":35273},{},[35274,35293],{"type":150,"tag":195,"props":35275,"children":35276},{},[35277,35281,35285,35289],{"type":150,"tag":405,"props":35278,"children":35279},{},[35280],{"type":155,"value":6923},{"type":150,"tag":417,"props":35282,"children":35283},{"stretchy":207},[35284],{"type":155,"value":972},{"type":150,"tag":405,"props":35286,"children":35287},{},[35288],{"type":155,"value":5591},{"type":150,"tag":417,"props":35290,"children":35291},{"stretchy":207},[35292],{"type":155,"value":982},{"type":150,"tag":223,"props":35294,"children":35295},{"encoding":225},[35296],{"type":155,"value":32183},{"type":150,"tag":169,"props":35298,"children":35300},{"className":35299,"ariaHidden":214},[232],[35301],{"type":150,"tag":169,"props":35302,"children":35304},{"className":35303},[237],[35305,35309,35314,35319,35324],{"type":150,"tag":169,"props":35306,"children":35308},{"className":35307,"style":999},[242],[],{"type":150,"tag":169,"props":35310,"children":35312},{"className":35311,"style":6968},[248,489],[35313],{"type":155,"value":6923},{"type":150,"tag":169,"props":35315,"children":35317},{"className":35316},[1004],[35318],{"type":155,"value":972},{"type":150,"tag":169,"props":35320,"children":35322},{"className":35321,"style":5780},[248,489],[35323],{"type":155,"value":5591},{"type":150,"tag":169,"props":35325,"children":35327},{"className":35326},[1015],[35328],{"type":155,"value":982},{"type":155,"value":35330}," 还是 ",{"type":150,"tag":169,"props":35332,"children":35334},{"className":35333},[177],[35335,35373],{"type":150,"tag":169,"props":35336,"children":35338},{"className":35337},[182],[35339],{"type":150,"tag":185,"props":35340,"children":35341},{"xmlns":187},[35342],{"type":150,"tag":191,"props":35343,"children":35344},{},[35345,35368],{"type":150,"tag":195,"props":35346,"children":35347},{},[35348,35352,35356,35360,35364],{"type":150,"tag":405,"props":35349,"children":35350},{},[35351],{"type":155,"value":6923},{"type":150,"tag":417,"props":35353,"children":35354},{"stretchy":207},[35355],{"type":155,"value":972},{"type":150,"tag":405,"props":35357,"children":35358},{},[35359],{"type":155,"value":5591},{"type":150,"tag":405,"props":35361,"children":35362},{},[35363],{"type":155,"value":24918},{"type":150,"tag":417,"props":35365,"children":35366},{"stretchy":207},[35367],{"type":155,"value":982},{"type":150,"tag":223,"props":35369,"children":35370},{"encoding":225},[35371],{"type":155,"value":35372},"O(NB)",{"type":150,"tag":169,"props":35374,"children":35376},{"className":35375,"ariaHidden":214},[232],[35377],{"type":150,"tag":169,"props":35378,"children":35380},{"className":35379},[237],[35381,35385,35390,35395,35400,35405],{"type":150,"tag":169,"props":35382,"children":35384},{"className":35383,"style":999},[242],[],{"type":150,"tag":169,"props":35386,"children":35388},{"className":35387,"style":6968},[248,489],[35389],{"type":155,"value":6923},{"type":150,"tag":169,"props":35391,"children":35393},{"className":35392},[1004],[35394],{"type":155,"value":972},{"type":150,"tag":169,"props":35396,"children":35398},{"className":35397,"style":5780},[248,489],[35399],{"type":155,"value":5591},{"type":150,"tag":169,"props":35401,"children":35403},{"className":35402,"style":27238},[248,489],[35404],{"type":155,"value":24918},{"type":150,"tag":169,"props":35406,"children":35408},{"className":35407},[1015],[35409],{"type":155,"value":982},{"type":155,"value":35411},"，而在更朴素的地方：",{"type":150,"tag":169,"props":35413,"children":35415},{"className":35414},[172],[35416],{"type":150,"tag":169,"props":35417,"children":35419},{"className":35418},[177],[35420,35455],{"type":150,"tag":169,"props":35421,"children":35423},{"className":35422},[182],[35424],{"type":150,"tag":185,"props":35425,"children":35426},{"xmlns":187,"display":188},[35427],{"type":150,"tag":191,"props":35428,"children":35429},{},[35430,35450],{"type":150,"tag":195,"props":35431,"children":35432},{},[35433],{"type":150,"tag":199,"props":35434,"children":35435},{"notation":201},[35436],{"type":150,"tag":204,"props":35437,"children":35438},{"scriptlevel":206,"displaystyle":207},[35439],{"type":150,"tag":204,"props":35440,"children":35441},{"scriptlevel":206,"displaystyle":207},[35442],{"type":150,"tag":204,"props":35443,"children":35444},{"scriptlevel":206,"displaystyle":214},[35445],{"type":150,"tag":217,"props":35446,"children":35447},{},[35448],{"type":155,"value":35449},"在同样的显存带宽和 FLOPs 下，谁还能准确找回 50 万 token 前那个关键细节。",{"type":150,"tag":223,"props":35451,"children":35452},{"encoding":225},[35453],{"type":155,"value":35454},"\\boxed{\\text{在同样的显存带宽和 FLOPs 下，谁还能准确找回 50 万 token 前那个关键细节。}}",{"type":150,"tag":169,"props":35456,"children":35458},{"className":35457,"ariaHidden":214},[232],[35459],{"type":150,"tag":169,"props":35460,"children":35462},{"className":35461},[237],[35463,35468],{"type":150,"tag":169,"props":35464,"children":35467},{"className":35465,"style":35466},[242],"height:1.3744em;vertical-align:-0.34em;",[],{"type":150,"tag":169,"props":35469,"children":35471},{"className":35470},[248],[35472],{"type":150,"tag":169,"props":35473,"children":35475},{"className":35474},[253,254],[35476,35564],{"type":150,"tag":169,"props":35477,"children":35479},{"className":35478},[259],[35480,35559],{"type":150,"tag":169,"props":35481,"children":35483},{"className":35482,"style":30668},[264],[35484,35548],{"type":150,"tag":169,"props":35485,"children":35486},{"style":35004},[35487,35491],{"type":150,"tag":169,"props":35488,"children":35490},{"className":35489,"style":35009},[274],[],{"type":150,"tag":169,"props":35492,"children":35494},{"className":35493},[280],[35495],{"type":150,"tag":169,"props":35496,"children":35498},{"className":35497},[248],[35499],{"type":150,"tag":169,"props":35500,"children":35502},{"className":35501},[248],[35503],{"type":150,"tag":169,"props":35504,"children":35506},{"className":35505},[248,155],[35507,35513,35519,35525,35531,35536,35542],{"type":150,"tag":169,"props":35508,"children":35510},{"className":35509},[248,297],[35511],{"type":155,"value":35512},"在同样的显存带宽和",{"type":150,"tag":169,"props":35514,"children":35516},{"className":35515},[248],[35517],{"type":155,"value":35518}," FLOPs ",{"type":150,"tag":169,"props":35520,"children":35522},{"className":35521},[248,297],[35523],{"type":155,"value":35524},"下，谁还能准确找回",{"type":150,"tag":169,"props":35526,"children":35528},{"className":35527},[248],[35529],{"type":155,"value":35530}," 50 ",{"type":150,"tag":169,"props":35532,"children":35534},{"className":35533},[248,297],[35535],{"type":155,"value":29680},{"type":150,"tag":169,"props":35537,"children":35539},{"className":35538},[248],[35540],{"type":155,"value":35541}," token ",{"type":150,"tag":169,"props":35543,"children":35545},{"className":35544},[248,297],[35546],{"type":155,"value":35547},"前那个关键细节。",{"type":150,"tag":169,"props":35549,"children":35550},{"style":30721},[35551,35555],{"type":150,"tag":169,"props":35552,"children":35554},{"className":35553,"style":35009},[274],[],{"type":150,"tag":169,"props":35556,"children":35558},{"className":35557,"style":35042},[348,349],[],{"type":150,"tag":169,"props":35560,"children":35562},{"className":35561},[355],[35563],{"type":155,"value":358},{"type":150,"tag":169,"props":35565,"children":35567},{"className":35566},[259],[35568],{"type":150,"tag":169,"props":35569,"children":35571},{"className":35570,"style":35056},[264],[35572],{"type":150,"tag":169,"props":35573,"children":35574},{},[],{"type":150,"tag":151,"props":35576,"children":35577},{},[35578],{"type":155,"value":35579},"这才是长上下文模型最难骗 benchmark 的地方。",{"type":150,"tag":372,"props":35581,"children":35583},{"id":35582},"l-n-d",[35584],{"type":155,"value":35585},"L × N × D",{"type":150,"tag":151,"props":35587,"children":35588},{},[35589],{"type":155,"value":35590},"把前面的方案合起来看，它们动的其实是同一个三维对象，也就是 KV 的形状：",{"type":150,"tag":169,"props":35592,"children":35594},{"className":35593},[172],[35595],{"type":150,"tag":169,"props":35596,"children":35598},{"className":35597},[177],[35599,35653],{"type":150,"tag":169,"props":35600,"children":35602},{"className":35601},[182],[35603],{"type":150,"tag":185,"props":35604,"children":35605},{"xmlns":187,"display":188},[35606],{"type":150,"tag":191,"props":35607,"children":35608},{},[35609,35648],{"type":150,"tag":195,"props":35610,"children":35611},{},[35612],{"type":150,"tag":199,"props":35613,"children":35614},{"notation":201},[35615],{"type":150,"tag":204,"props":35616,"children":35617},{"scriptlevel":206,"displaystyle":207},[35618],{"type":150,"tag":204,"props":35619,"children":35620},{"scriptlevel":206,"displaystyle":207},[35621],{"type":150,"tag":204,"props":35622,"children":35623},{"scriptlevel":206,"displaystyle":214},[35624],{"type":150,"tag":195,"props":35625,"children":35626},{},[35627,35631,35635,35639,35643],{"type":150,"tag":405,"props":35628,"children":35629},{},[35630],{"type":155,"value":5559},{"type":150,"tag":417,"props":35632,"children":35633},{},[35634],{"type":155,"value":5554},{"type":150,"tag":405,"props":35636,"children":35637},{},[35638],{"type":155,"value":5591},{"type":150,"tag":417,"props":35640,"children":35641},{},[35642],{"type":155,"value":5554},{"type":150,"tag":405,"props":35644,"children":35645},{},[35646],{"type":155,"value":35647},"D",{"type":150,"tag":223,"props":35649,"children":35650},{"encoding":225},[35651],{"type":155,"value":35652},"\\boxed{L \\times N \\times D}",{"type":150,"tag":169,"props":35654,"children":35656},{"className":35655,"ariaHidden":214},[232],[35657],{"type":150,"tag":169,"props":35658,"children":35660},{"className":35659},[237],[35661,35666],{"type":150,"tag":169,"props":35662,"children":35665},{"className":35663,"style":35664},[242],"height:1.4467em;vertical-align:-0.4233em;",[],{"type":150,"tag":169,"props":35667,"children":35669},{"className":35668},[248],[35670],{"type":150,"tag":169,"props":35671,"children":35673},{"className":35672},[253,254],[35674,35761],{"type":150,"tag":169,"props":35675,"children":35677},{"className":35676},[259],[35678,35756],{"type":150,"tag":169,"props":35679,"children":35681},{"className":35680,"style":265},[264],[35682,35744],{"type":150,"tag":169,"props":35683,"children":35685},{"style":35684},"top:-3.4467em;",[35686,35691],{"type":150,"tag":169,"props":35687,"children":35690},{"className":35688,"style":35689},[274],"height:3.4467em;",[],{"type":150,"tag":169,"props":35692,"children":35694},{"className":35693},[280],[35695],{"type":150,"tag":169,"props":35696,"children":35698},{"className":35697},[248],[35699],{"type":150,"tag":169,"props":35700,"children":35702},{"className":35701},[248],[35703,35708,35712,35717,35721,35726,35730,35735,35739],{"type":150,"tag":169,"props":35704,"children":35706},{"className":35705},[248,489],[35707],{"type":155,"value":5559},{"type":150,"tag":169,"props":35709,"children":35711},{"className":35710,"style":5167},[558],[],{"type":150,"tag":169,"props":35713,"children":35715},{"className":35714},[5172],[35716],{"type":155,"value":5554},{"type":150,"tag":169,"props":35718,"children":35720},{"className":35719,"style":5167},[558],[],{"type":150,"tag":169,"props":35722,"children":35724},{"className":35723,"style":5780},[248,489],[35725],{"type":155,"value":5591},{"type":150,"tag":169,"props":35727,"children":35729},{"className":35728,"style":5167},[558],[],{"type":150,"tag":169,"props":35731,"children":35733},{"className":35732},[5172],[35734],{"type":155,"value":5554},{"type":150,"tag":169,"props":35736,"children":35738},{"className":35737,"style":5167},[558],[],{"type":150,"tag":169,"props":35740,"children":35742},{"className":35741,"style":6968},[248,489],[35743],{"type":155,"value":35647},{"type":150,"tag":169,"props":35745,"children":35746},{"style":339},[35747,35751],{"type":150,"tag":169,"props":35748,"children":35750},{"className":35749,"style":35689},[274],[],{"type":150,"tag":169,"props":35752,"children":35755},{"className":35753,"style":35754},[348,349],"height:1.4467em;border-style:solid;border-width:0.04em;",[],{"type":150,"tag":169,"props":35757,"children":35759},{"className":35758},[355],[35760],{"type":155,"value":358},{"type":150,"tag":169,"props":35762,"children":35764},{"className":35763},[259],[35765],{"type":150,"tag":169,"props":35766,"children":35769},{"className":35767,"style":35768},[264],"height:0.4233em;",[35770],{"type":150,"tag":169,"props":35771,"children":35772},{},[],{"type":150,"tag":151,"props":35774,"children":35775},{},[35776,35818,35819,35861,35863,35905],{"type":150,"tag":169,"props":35777,"children":35779},{"className":35778},[177],[35780,35801],{"type":150,"tag":169,"props":35781,"children":35783},{"className":35782},[182],[35784],{"type":150,"tag":185,"props":35785,"children":35786},{"xmlns":187},[35787],{"type":150,"tag":191,"props":35788,"children":35789},{},[35790,35797],{"type":150,"tag":195,"props":35791,"children":35792},{},[35793],{"type":150,"tag":405,"props":35794,"children":35795},{},[35796],{"type":155,"value":5559},{"type":150,"tag":223,"props":35798,"children":35799},{"encoding":225},[35800],{"type":155,"value":5559},{"type":150,"tag":169,"props":35802,"children":35804},{"className":35803,"ariaHidden":214},[232],[35805],{"type":150,"tag":169,"props":35806,"children":35808},{"className":35807},[237],[35809,35813],{"type":150,"tag":169,"props":35810,"children":35812},{"className":35811,"style":5854},[242],[],{"type":150,"tag":169,"props":35814,"children":35816},{"className":35815},[248,489],[35817],{"type":155,"value":5559},{"type":155,"value":5862},{"type":150,"tag":169,"props":35820,"children":35822},{"className":35821},[177],[35823,35844],{"type":150,"tag":169,"props":35824,"children":35826},{"className":35825},[182],[35827],{"type":150,"tag":185,"props":35828,"children":35829},{"xmlns":187},[35830],{"type":150,"tag":191,"props":35831,"children":35832},{},[35833,35840],{"type":150,"tag":195,"props":35834,"children":35835},{},[35836],{"type":150,"tag":405,"props":35837,"children":35838},{},[35839],{"type":155,"value":5591},{"type":150,"tag":223,"props":35841,"children":35842},{"encoding":225},[35843],{"type":155,"value":5591},{"type":150,"tag":169,"props":35845,"children":35847},{"className":35846,"ariaHidden":214},[232],[35848],{"type":150,"tag":169,"props":35849,"children":35851},{"className":35850},[237],[35852,35856],{"type":150,"tag":169,"props":35853,"children":35855},{"className":35854,"style":5854},[242],[],{"type":150,"tag":169,"props":35857,"children":35859},{"className":35858,"style":5780},[248,489],[35860],{"type":155,"value":5591},{"type":155,"value":35862}," 是序列长度，",{"type":150,"tag":169,"props":35864,"children":35866},{"className":35865},[177],[35867,35888],{"type":150,"tag":169,"props":35868,"children":35870},{"className":35869},[182],[35871],{"type":150,"tag":185,"props":35872,"children":35873},{"xmlns":187},[35874],{"type":150,"tag":191,"props":35875,"children":35876},{},[35877,35884],{"type":150,"tag":195,"props":35878,"children":35879},{},[35880],{"type":150,"tag":405,"props":35881,"children":35882},{},[35883],{"type":155,"value":35647},{"type":150,"tag":223,"props":35885,"children":35886},{"encoding":225},[35887],{"type":155,"value":35647},{"type":150,"tag":169,"props":35889,"children":35891},{"className":35890,"ariaHidden":214},[232],[35892],{"type":150,"tag":169,"props":35893,"children":35895},{"className":35894},[237],[35896,35900],{"type":150,"tag":169,"props":35897,"children":35899},{"className":35898,"style":5854},[242],[],{"type":150,"tag":169,"props":35901,"children":35903},{"className":35902,"style":6968},[248,489],[35904],{"type":155,"value":35647},{"type":155,"value":35906}," 是每个 token 存下来的表示宽度。于是整条 DeepSeek 路线可以重新写成：",{"type":150,"tag":169,"props":35908,"children":35910},{"className":35909},[172],[35911],{"type":150,"tag":169,"props":35912,"children":35914},{"className":35913},[177],[35915,35942],{"type":150,"tag":169,"props":35916,"children":35918},{"className":35917},[182],[35919],{"type":150,"tag":185,"props":35920,"children":35921},{"xmlns":187,"display":188},[35922],{"type":150,"tag":191,"props":35923,"children":35924},{},[35925,35937],{"type":150,"tag":195,"props":35926,"children":35927},{},[35928,35933],{"type":150,"tag":217,"props":35929,"children":35930},{},[35931],{"type":155,"value":35932},"GQA \u002F MLA：压 ",{"type":150,"tag":405,"props":35934,"children":35935},{},[35936],{"type":155,"value":35647},{"type":150,"tag":223,"props":35938,"children":35939},{"encoding":225},[35940],{"type":155,"value":35941},"\\text{GQA \u002F MLA：压 } D",{"type":150,"tag":169,"props":35943,"children":35945},{"className":35944,"ariaHidden":214},[232],[35946],{"type":150,"tag":169,"props":35947,"children":35949},{"className":35948},[237],[35950,35954,35975],{"type":150,"tag":169,"props":35951,"children":35953},{"className":35952,"style":999},[242],[],{"type":150,"tag":169,"props":35955,"children":35957},{"className":35956},[248,155],[35958,35964,35970],{"type":150,"tag":169,"props":35959,"children":35961},{"className":35960},[248],[35962],{"type":155,"value":35963},"GQA \u002F MLA",{"type":150,"tag":169,"props":35965,"children":35967},{"className":35966},[248,297],[35968],{"type":155,"value":35969},"：压",{"type":150,"tag":169,"props":35971,"children":35973},{"className":35972},[248],[35974],{"type":155,"value":8763},{"type":150,"tag":169,"props":35976,"children":35978},{"className":35977,"style":6968},[248,489],[35979],{"type":155,"value":35647},{"type":150,"tag":169,"props":35981,"children":35983},{"className":35982},[172],[35984],{"type":150,"tag":169,"props":35985,"children":35987},{"className":35986},[177],[35988,36015],{"type":150,"tag":169,"props":35989,"children":35991},{"className":35990},[182],[35992],{"type":150,"tag":185,"props":35993,"children":35994},{"xmlns":187,"display":188},[35995],{"type":150,"tag":191,"props":35996,"children":35997},{},[35998,36010],{"type":150,"tag":195,"props":35999,"children":36000},{},[36001,36006],{"type":150,"tag":217,"props":36002,"children":36003},{},[36004],{"type":155,"value":36005},"V4（CSA \u002F HCA）：压 ",{"type":150,"tag":405,"props":36007,"children":36008},{},[36009],{"type":155,"value":5591},{"type":150,"tag":223,"props":36011,"children":36012},{"encoding":225},[36013],{"type":155,"value":36014},"\\text{V4（CSA \u002F HCA）：压 } N",{"type":150,"tag":169,"props":36016,"children":36018},{"className":36017,"ariaHidden":214},[232],[36019],{"type":150,"tag":169,"props":36020,"children":36022},{"className":36021},[237],[36023,36027,36060],{"type":150,"tag":169,"props":36024,"children":36026},{"className":36025,"style":999},[242],[],{"type":150,"tag":169,"props":36028,"children":36030},{"className":36029},[248,155],[36031,36037,36043,36049,36055],{"type":150,"tag":169,"props":36032,"children":36034},{"className":36033},[248],[36035],{"type":155,"value":36036},"V4",{"type":150,"tag":169,"props":36038,"children":36040},{"className":36039},[248,297],[36041],{"type":155,"value":36042},"（",{"type":150,"tag":169,"props":36044,"children":36046},{"className":36045},[248],[36047],{"type":155,"value":36048},"CSA \u002F HCA",{"type":150,"tag":169,"props":36050,"children":36052},{"className":36051},[248,297],[36053],{"type":155,"value":36054},"）：压",{"type":150,"tag":169,"props":36056,"children":36058},{"className":36057},[248],[36059],{"type":155,"value":8763},{"type":150,"tag":169,"props":36061,"children":36063},{"className":36062,"style":5780},[248,489],[36064],{"type":155,"value":5591},{"type":150,"tag":169,"props":36066,"children":36068},{"className":36067},[172],[36069],{"type":150,"tag":169,"props":36070,"children":36072},{"className":36071},[177],[36073,36105],{"type":150,"tag":169,"props":36074,"children":36076},{"className":36075},[182],[36077],{"type":150,"tag":185,"props":36078,"children":36079},{"xmlns":187,"display":188},[36080],{"type":150,"tag":191,"props":36081,"children":36082},{},[36083,36100],{"type":150,"tag":195,"props":36084,"children":36085},{},[36086,36091,36095],{"type":150,"tag":217,"props":36087,"children":36088},{},[36089],{"type":155,"value":36090},"V4.1：压 ",{"type":150,"tag":405,"props":36092,"children":36093},{},[36094],{"type":155,"value":5559},{"type":150,"tag":217,"props":36096,"children":36097},{},[36098],{"type":155,"value":36099}," 和 compute path",{"type":150,"tag":223,"props":36101,"children":36102},{"encoding":225},[36103],{"type":155,"value":36104},"\\text{V4.1：压 } L \\text{ 和 compute path}",{"type":150,"tag":169,"props":36106,"children":36108},{"className":36107,"ariaHidden":214},[232],[36109],{"type":150,"tag":169,"props":36110,"children":36112},{"className":36111},[237],[36113,36117,36137,36142],{"type":150,"tag":169,"props":36114,"children":36116},{"className":36115,"style":1121},[242],[],{"type":150,"tag":169,"props":36118,"children":36120},{"className":36119},[248,155],[36121,36127,36132],{"type":150,"tag":169,"props":36122,"children":36124},{"className":36123},[248],[36125],{"type":155,"value":36126},"V4.1",{"type":150,"tag":169,"props":36128,"children":36130},{"className":36129},[248,297],[36131],{"type":155,"value":35969},{"type":150,"tag":169,"props":36133,"children":36135},{"className":36134},[248],[36136],{"type":155,"value":8763},{"type":150,"tag":169,"props":36138,"children":36140},{"className":36139},[248,489],[36141],{"type":155,"value":5559},{"type":150,"tag":169,"props":36143,"children":36145},{"className":36144},[248,155],[36146,36151,36156],{"type":150,"tag":169,"props":36147,"children":36149},{"className":36148},[248],[36150],{"type":155,"value":8763},{"type":150,"tag":169,"props":36152,"children":36154},{"className":36153},[248,297],[36155],{"type":155,"value":324},{"type":150,"tag":169,"props":36157,"children":36159},{"className":36158},[248],[36160],{"type":155,"value":36161}," compute path",{"type":150,"tag":151,"props":36163,"children":36164},{},[36165],{"type":155,"value":36166},"到 V4.1，问题变成了：",{"type":150,"tag":169,"props":36168,"children":36170},{"className":36169},[172],[36171],{"type":150,"tag":169,"props":36172,"children":36174},{"className":36173},[177],[36175,36210],{"type":150,"tag":169,"props":36176,"children":36178},{"className":36177},[182],[36179],{"type":150,"tag":185,"props":36180,"children":36181},{"xmlns":187,"display":188},[36182],{"type":150,"tag":191,"props":36183,"children":36184},{},[36185,36205],{"type":150,"tag":195,"props":36186,"children":36187},{},[36188],{"type":150,"tag":199,"props":36189,"children":36190},{"notation":201},[36191],{"type":150,"tag":204,"props":36192,"children":36193},{"scriptlevel":206,"displaystyle":207},[36194],{"type":150,"tag":204,"props":36195,"children":36196},{"scriptlevel":206,"displaystyle":207},[36197],{"type":150,"tag":204,"props":36198,"children":36199},{"scriptlevel":206,"displaystyle":214},[36200],{"type":150,"tag":217,"props":36201,"children":36202},{},[36203],{"type":155,"value":36204},"既然历史已经压缩过一次，为什么每一层还要重复为它付费？",{"type":150,"tag":223,"props":36206,"children":36207},{"encoding":225},[36208],{"type":155,"value":36209},"\\boxed{\\text{既然历史已经压缩过一次，为什么每一层还要重复为它付费？}}",{"type":150,"tag":169,"props":36211,"children":36213},{"className":36212,"ariaHidden":214},[232],[36214],{"type":150,"tag":169,"props":36215,"children":36217},{"className":36216},[237],[36218,36223],{"type":150,"tag":169,"props":36219,"children":36222},{"className":36220,"style":36221},[242],"height:1.3633em;vertical-align:-0.34em;",[],{"type":150,"tag":169,"props":36224,"children":36226},{"className":36225},[248],[36227],{"type":150,"tag":169,"props":36228,"children":36230},{"className":36229},[253,254],[36231,36286],{"type":150,"tag":169,"props":36232,"children":36234},{"className":36233},[259],[36235,36281],{"type":150,"tag":169,"props":36236,"children":36238},{"className":36237,"style":265},[264],[36239,36269],{"type":150,"tag":169,"props":36240,"children":36242},{"style":36241},"top:-3.3633em;",[36243,36248],{"type":150,"tag":169,"props":36244,"children":36247},{"className":36245,"style":36246},[274],"height:3.3633em;",[],{"type":150,"tag":169,"props":36249,"children":36251},{"className":36250},[280],[36252],{"type":150,"tag":169,"props":36253,"children":36255},{"className":36254},[248],[36256],{"type":150,"tag":169,"props":36257,"children":36259},{"className":36258},[248],[36260],{"type":150,"tag":169,"props":36261,"children":36263},{"className":36262},[248,155],[36264],{"type":150,"tag":169,"props":36265,"children":36267},{"className":36266},[248,297],[36268],{"type":155,"value":36204},{"type":150,"tag":169,"props":36270,"children":36271},{"style":339},[36272,36276],{"type":150,"tag":169,"props":36273,"children":36275},{"className":36274,"style":36246},[274],[],{"type":150,"tag":169,"props":36277,"children":36280},{"className":36278,"style":36279},[348,349],"height:1.3633em;border-style:solid;border-width:0.04em;",[],{"type":150,"tag":169,"props":36282,"children":36284},{"className":36283},[355],[36285],{"type":155,"value":358},{"type":150,"tag":169,"props":36287,"children":36289},{"className":36288},[259],[36290],{"type":150,"tag":169,"props":36291,"children":36293},{"className":36292,"style":35056},[264],[36294],{"type":150,"tag":169,"props":36295,"children":36296},{},[],{"type":150,"tag":151,"props":36298,"children":36299},{},[36300,36302,36344,36346,36388,36390,36432,36434,36439],{"type":155,"value":36301},"ds4.1f 采用的是一个新架构族：552B MoE，Causal-Encoder-Decoder，输入侧约 8B activated、输出侧约 16B activated。这个不对称的计算预算是关键。要理解为什么 8B 和 16B 是分开的两个数，得把那两个瓶颈各自的账算一遍。先定符号。",{"type":150,"tag":169,"props":36303,"children":36305},{"className":36304},[177],[36306,36327],{"type":150,"tag":169,"props":36307,"children":36309},{"className":36308},[182],[36310],{"type":150,"tag":185,"props":36311,"children":36312},{"xmlns":187},[36313],{"type":150,"tag":191,"props":36314,"children":36315},{},[36316,36323],{"type":150,"tag":195,"props":36317,"children":36318},{},[36319],{"type":150,"tag":405,"props":36320,"children":36321},{},[36322],{"type":155,"value":25172},{"type":150,"tag":223,"props":36324,"children":36325},{"encoding":225},[36326],{"type":155,"value":25172},{"type":150,"tag":169,"props":36328,"children":36330},{"className":36329,"ariaHidden":214},[232],[36331],{"type":150,"tag":169,"props":36332,"children":36334},{"className":36333},[237],[36335,36339],{"type":150,"tag":169,"props":36336,"children":36338},{"className":36337,"style":5854},[242],[],{"type":150,"tag":169,"props":36340,"children":36342},{"className":36341,"style":3869},[248,489],[36343],{"type":155,"value":25172},{"type":155,"value":36345}," 是参数量（MoE 里指 activated 部分），",{"type":150,"tag":169,"props":36347,"children":36349},{"className":36348},[177],[36350,36371],{"type":150,"tag":169,"props":36351,"children":36353},{"className":36352},[182],[36354],{"type":150,"tag":185,"props":36355,"children":36356},{"xmlns":187},[36357],{"type":150,"tag":191,"props":36358,"children":36359},{},[36360,36367],{"type":150,"tag":195,"props":36361,"children":36362},{},[36363],{"type":150,"tag":405,"props":36364,"children":36365},{},[36366],{"type":155,"value":22264},{"type":150,"tag":223,"props":36368,"children":36369},{"encoding":225},[36370],{"type":155,"value":22264},{"type":150,"tag":169,"props":36372,"children":36374},{"className":36373,"ariaHidden":214},[232],[36375],{"type":150,"tag":169,"props":36376,"children":36378},{"className":36377},[237],[36379,36383],{"type":150,"tag":169,"props":36380,"children":36382},{"className":36381,"style":942},[242],[],{"type":150,"tag":169,"props":36384,"children":36386},{"className":36385},[248,489],[36387],{"type":155,"value":22264},{"type":155,"value":36389}," 是每个参数的字节数，",{"type":150,"tag":169,"props":36391,"children":36393},{"className":36392},[177],[36394,36415],{"type":150,"tag":169,"props":36395,"children":36397},{"className":36396},[182],[36398],{"type":150,"tag":185,"props":36399,"children":36400},{"xmlns":187},[36401],{"type":150,"tag":191,"props":36402,"children":36403},{},[36404,36411],{"type":150,"tag":195,"props":36405,"children":36406},{},[36407],{"type":150,"tag":405,"props":36408,"children":36409},{},[36410],{"type":155,"value":5591},{"type":150,"tag":223,"props":36412,"children":36413},{"encoding":225},[36414],{"type":155,"value":5591},{"type":150,"tag":169,"props":36416,"children":36418},{"className":36417,"ariaHidden":214},[232],[36419],{"type":150,"tag":169,"props":36420,"children":36422},{"className":36421},[237],[36423,36427],{"type":150,"tag":169,"props":36424,"children":36426},{"className":36425,"style":5854},[242],[],{"type":150,"tag":169,"props":36428,"children":36430},{"className":36429,"style":5780},[248,489],[36431],{"type":155,"value":5591},{"type":155,"value":36433}," 是序列长度。",{"type":150,"tag":158,"props":36435,"children":36436},{},[36437],{"type":155,"value":36438},"输入侧",{"type":155,"value":36440},"面对的是长 prompt。",{"type":150,"tag":151,"props":36442,"children":36443},{},[36444,36446,36507,36509,36586],{"type":155,"value":36445},"1M 个 token 全部要走一遍编码器，计算量约 ",{"type":150,"tag":169,"props":36447,"children":36449},{"className":36448},[177],[36450,36480],{"type":150,"tag":169,"props":36451,"children":36453},{"className":36452},[182],[36454],{"type":150,"tag":185,"props":36455,"children":36456},{"xmlns":187},[36457],{"type":150,"tag":191,"props":36458,"children":36459},{},[36460,36475],{"type":150,"tag":195,"props":36461,"children":36462},{},[36463,36467,36471],{"type":150,"tag":411,"props":36464,"children":36465},{},[36466],{"type":155,"value":433},{"type":150,"tag":405,"props":36468,"children":36469},{},[36470],{"type":155,"value":25172},{"type":150,"tag":405,"props":36472,"children":36473},{},[36474],{"type":155,"value":5591},{"type":150,"tag":223,"props":36476,"children":36477},{"encoding":225},[36478],{"type":155,"value":36479},"2PN",{"type":150,"tag":169,"props":36481,"children":36483},{"className":36482,"ariaHidden":214},[232],[36484],{"type":150,"tag":169,"props":36485,"children":36487},{"className":36486},[237],[36488,36492,36497,36502],{"type":150,"tag":169,"props":36489,"children":36491},{"className":36490,"style":5854},[242],[],{"type":150,"tag":169,"props":36493,"children":36495},{"className":36494},[248],[36496],{"type":155,"value":433},{"type":150,"tag":169,"props":36498,"children":36500},{"className":36499,"style":3869},[248,489],[36501],{"type":155,"value":25172},{"type":150,"tag":169,"props":36503,"children":36505},{"className":36504,"style":5780},[248,489],[36506],{"type":155,"value":5591},{"type":155,"value":36508},"，而要读的权重只有 ",{"type":150,"tag":169,"props":36510,"children":36512},{"className":36511},[177],[36513,36543],{"type":150,"tag":169,"props":36514,"children":36516},{"className":36515},[182],[36517],{"type":150,"tag":185,"props":36518,"children":36519},{"xmlns":187},[36520],{"type":150,"tag":191,"props":36521,"children":36522},{},[36523,36538],{"type":150,"tag":195,"props":36524,"children":36525},{},[36526,36530,36534],{"type":150,"tag":405,"props":36527,"children":36528},{},[36529],{"type":155,"value":25172},{"type":150,"tag":417,"props":36531,"children":36532},{},[36533],{"type":155,"value":5554},{"type":150,"tag":405,"props":36535,"children":36536},{},[36537],{"type":155,"value":22264},{"type":150,"tag":223,"props":36539,"children":36540},{"encoding":225},[36541],{"type":155,"value":36542},"P \\times b",{"type":150,"tag":169,"props":36544,"children":36546},{"className":36545,"ariaHidden":214},[232],[36547,36573],{"type":150,"tag":169,"props":36548,"children":36550},{"className":36549},[237],[36551,36555,36560,36564,36569],{"type":150,"tag":169,"props":36552,"children":36554},{"className":36553,"style":5644},[242],[],{"type":150,"tag":169,"props":36556,"children":36558},{"className":36557,"style":3869},[248,489],[36559],{"type":155,"value":25172},{"type":150,"tag":169,"props":36561,"children":36563},{"className":36562,"style":5167},[558],[],{"type":150,"tag":169,"props":36565,"children":36567},{"className":36566},[5172],[36568],{"type":155,"value":5554},{"type":150,"tag":169,"props":36570,"children":36572},{"className":36571,"style":5167},[558],[],{"type":150,"tag":169,"props":36574,"children":36576},{"className":36575},[237],[36577,36581],{"type":150,"tag":169,"props":36578,"children":36580},{"className":36579,"style":942},[242],[],{"type":150,"tag":169,"props":36582,"children":36584},{"className":36583},[248,489],[36585],{"type":155,"value":22264},{"type":155,"value":904},{"type":150,"tag":151,"props":36588,"children":36589},{},[36590,36632,36634,36676,36678,36683,36685,36690,36692,36743,36745,36821],{"type":150,"tag":169,"props":36591,"children":36593},{"className":36592},[177],[36594,36615],{"type":150,"tag":169,"props":36595,"children":36597},{"className":36596},[182],[36598],{"type":150,"tag":185,"props":36599,"children":36600},{"xmlns":187},[36601],{"type":150,"tag":191,"props":36602,"children":36603},{},[36604,36611],{"type":150,"tag":195,"props":36605,"children":36606},{},[36607],{"type":150,"tag":405,"props":36608,"children":36609},{},[36610],{"type":155,"value":5591},{"type":150,"tag":223,"props":36612,"children":36613},{"encoding":225},[36614],{"type":155,"value":5591},{"type":150,"tag":169,"props":36616,"children":36618},{"className":36617,"ariaHidden":214},[232],[36619],{"type":150,"tag":169,"props":36620,"children":36622},{"className":36621},[237],[36623,36627],{"type":150,"tag":169,"props":36624,"children":36626},{"className":36625,"style":5854},[242],[],{"type":150,"tag":169,"props":36628,"children":36630},{"className":36629,"style":5780},[248,489],[36631],{"type":155,"value":5591},{"type":155,"value":36633}," 到百万量级时，算术强度能到几千 FLOPs\u002Fbyte，早就超过了硬件的算力带宽比。这一侧完全被算力卡住。想在 prefill 上省钱，唯一有效的办法就是把 ",{"type":150,"tag":169,"props":36635,"children":36637},{"className":36636},[177],[36638,36659],{"type":150,"tag":169,"props":36639,"children":36641},{"className":36640},[182],[36642],{"type":150,"tag":185,"props":36643,"children":36644},{"xmlns":187},[36645],{"type":150,"tag":191,"props":36646,"children":36647},{},[36648,36655],{"type":150,"tag":195,"props":36649,"children":36650},{},[36651],{"type":150,"tag":405,"props":36652,"children":36653},{},[36654],{"type":155,"value":25172},{"type":150,"tag":223,"props":36656,"children":36657},{"encoding":225},[36658],{"type":155,"value":25172},{"type":150,"tag":169,"props":36660,"children":36662},{"className":36661,"ariaHidden":214},[232],[36663],{"type":150,"tag":169,"props":36664,"children":36666},{"className":36665},[237],[36667,36671],{"type":150,"tag":169,"props":36668,"children":36670},{"className":36669,"style":5854},[242],[],{"type":150,"tag":169,"props":36672,"children":36674},{"className":36673,"style":3869},[248,489],[36675],{"type":155,"value":25172},{"type":155,"value":36677}," 压下去。",{"type":150,"tag":158,"props":36679,"children":36680},{},[36681],{"type":155,"value":36682},"8B 是这么来的。",{"type":155,"value":36684}," ",{"type":150,"tag":158,"props":36686,"children":36687},{},[36688],{"type":155,"value":36689},"输出侧",{"type":155,"value":36691},"面对的是逐个生成的 token，情况正好反过来。每生成一个 token，计算量只有 ",{"type":150,"tag":169,"props":36693,"children":36695},{"className":36694},[177],[36696,36721],{"type":150,"tag":169,"props":36697,"children":36699},{"className":36698},[182],[36700],{"type":150,"tag":185,"props":36701,"children":36702},{"xmlns":187},[36703],{"type":150,"tag":191,"props":36704,"children":36705},{},[36706,36717],{"type":150,"tag":195,"props":36707,"children":36708},{},[36709,36713],{"type":150,"tag":411,"props":36710,"children":36711},{},[36712],{"type":155,"value":433},{"type":150,"tag":405,"props":36714,"children":36715},{},[36716],{"type":155,"value":25172},{"type":150,"tag":223,"props":36718,"children":36719},{"encoding":225},[36720],{"type":155,"value":25661},{"type":150,"tag":169,"props":36722,"children":36724},{"className":36723,"ariaHidden":214},[232],[36725],{"type":150,"tag":169,"props":36726,"children":36728},{"className":36727},[237],[36729,36733,36738],{"type":150,"tag":169,"props":36730,"children":36732},{"className":36731,"style":5854},[242],[],{"type":150,"tag":169,"props":36734,"children":36736},{"className":36735},[248],[36737],{"type":155,"value":433},{"type":150,"tag":169,"props":36739,"children":36741},{"className":36740,"style":3869},[248,489],[36742],{"type":155,"value":25172},{"type":155,"value":36744},"，但要读两份东西：模型权重 ",{"type":150,"tag":169,"props":36746,"children":36748},{"className":36747},[177],[36749,36778],{"type":150,"tag":169,"props":36750,"children":36752},{"className":36751},[182],[36753],{"type":150,"tag":185,"props":36754,"children":36755},{"xmlns":187},[36756],{"type":150,"tag":191,"props":36757,"children":36758},{},[36759,36774],{"type":150,"tag":195,"props":36760,"children":36761},{},[36762,36766,36770],{"type":150,"tag":405,"props":36763,"children":36764},{},[36765],{"type":155,"value":25172},{"type":150,"tag":417,"props":36767,"children":36768},{},[36769],{"type":155,"value":5554},{"type":150,"tag":405,"props":36771,"children":36772},{},[36773],{"type":155,"value":22264},{"type":150,"tag":223,"props":36775,"children":36776},{"encoding":225},[36777],{"type":155,"value":36542},{"type":150,"tag":169,"props":36779,"children":36781},{"className":36780,"ariaHidden":214},[232],[36782,36808],{"type":150,"tag":169,"props":36783,"children":36785},{"className":36784},[237],[36786,36790,36795,36799,36804],{"type":150,"tag":169,"props":36787,"children":36789},{"className":36788,"style":5644},[242],[],{"type":150,"tag":169,"props":36791,"children":36793},{"className":36792,"style":3869},[248,489],[36794],{"type":155,"value":25172},{"type":150,"tag":169,"props":36796,"children":36798},{"className":36797,"style":5167},[558],[],{"type":150,"tag":169,"props":36800,"children":36802},{"className":36801},[5172],[36803],{"type":155,"value":5554},{"type":150,"tag":169,"props":36805,"children":36807},{"className":36806,"style":5167},[558],[],{"type":150,"tag":169,"props":36809,"children":36811},{"className":36810},[237],[36812,36816],{"type":150,"tag":169,"props":36813,"children":36815},{"className":36814,"style":942},[242],[],{"type":150,"tag":169,"props":36817,"children":36819},{"className":36818},[248,489],[36820],{"type":155,"value":22264},{"type":155,"value":36822},"，加上 KV Cache。而模型权重是必须整体读一遍的。",{"type":150,"tag":151,"props":36824,"children":36825},{},[36826],{"type":155,"value":36827},"拿 16B activated、BF16 存储来算：",{"type":150,"tag":169,"props":36829,"children":36831},{"className":36830},[172],[36832],{"type":150,"tag":169,"props":36833,"children":36835},{"className":36834},[177],[36836,36889],{"type":150,"tag":169,"props":36837,"children":36839},{"className":36838},[182],[36840],{"type":150,"tag":185,"props":36841,"children":36842},{"xmlns":187,"display":188},[36843],{"type":150,"tag":191,"props":36844,"children":36845},{},[36846,36884],{"type":150,"tag":195,"props":36847,"children":36848},{},[36849,36854,36858,36862,36866,36871,36875,36880],{"type":150,"tag":411,"props":36850,"children":36851},{},[36852],{"type":155,"value":36853},"16",{"type":150,"tag":217,"props":36855,"children":36856},{},[36857],{"type":155,"value":24918},{"type":150,"tag":417,"props":36859,"children":36860},{},[36861],{"type":155,"value":5554},{"type":150,"tag":411,"props":36863,"children":36864},{},[36865],{"type":155,"value":433},{"type":150,"tag":217,"props":36867,"children":36868},{},[36869],{"type":155,"value":36870}," bytes",{"type":150,"tag":417,"props":36872,"children":36873},{},[36874],{"type":155,"value":1045},{"type":150,"tag":411,"props":36876,"children":36877},{},[36878],{"type":155,"value":36879},"32",{"type":150,"tag":217,"props":36881,"children":36882},{},[36883],{"type":155,"value":29467},{"type":150,"tag":223,"props":36885,"children":36886},{"encoding":225},[36887],{"type":155,"value":36888},"16\\text{B} \\times 2\\ \\text{bytes} = 32\\ \\text{GB}",{"type":150,"tag":169,"props":36890,"children":36892},{"className":36891,"ariaHidden":214},[232],[36893,36928,36968],{"type":150,"tag":169,"props":36894,"children":36896},{"className":36895},[237],[36897,36901,36906,36915,36919,36924],{"type":150,"tag":169,"props":36898,"children":36900},{"className":36899,"style":5644},[242],[],{"type":150,"tag":169,"props":36902,"children":36904},{"className":36903},[248],[36905],{"type":155,"value":36853},{"type":150,"tag":169,"props":36907,"children":36909},{"className":36908},[248,155],[36910],{"type":150,"tag":169,"props":36911,"children":36913},{"className":36912},[248],[36914],{"type":155,"value":24918},{"type":150,"tag":169,"props":36916,"children":36918},{"className":36917,"style":5167},[558],[],{"type":150,"tag":169,"props":36920,"children":36922},{"className":36921},[5172],[36923],{"type":155,"value":5554},{"type":150,"tag":169,"props":36925,"children":36927},{"className":36926,"style":5167},[558],[],{"type":150,"tag":169,"props":36929,"children":36931},{"className":36930},[237],[36932,36936,36941,36946,36955,36959,36964],{"type":150,"tag":169,"props":36933,"children":36935},{"className":36934,"style":1121},[242],[],{"type":150,"tag":169,"props":36937,"children":36939},{"className":36938},[248],[36940],{"type":155,"value":433},{"type":150,"tag":169,"props":36942,"children":36944},{"className":36943},[558],[36945],{"type":155,"value":8763},{"type":150,"tag":169,"props":36947,"children":36949},{"className":36948},[248,155],[36950],{"type":150,"tag":169,"props":36951,"children":36953},{"className":36952},[248],[36954],{"type":155,"value":5600},{"type":150,"tag":169,"props":36956,"children":36958},{"className":36957,"style":839},[558],[],{"type":150,"tag":169,"props":36960,"children":36962},{"className":36961},[844],[36963],{"type":155,"value":1045},{"type":150,"tag":169,"props":36965,"children":36967},{"className":36966,"style":839},[558],[],{"type":150,"tag":169,"props":36969,"children":36971},{"className":36970},[237],[36972,36976,36981,36986],{"type":150,"tag":169,"props":36973,"children":36975},{"className":36974,"style":5854},[242],[],{"type":150,"tag":169,"props":36977,"children":36979},{"className":36978},[248],[36980],{"type":155,"value":36879},{"type":150,"tag":169,"props":36982,"children":36984},{"className":36983},[558],[36985],{"type":155,"value":8763},{"type":150,"tag":169,"props":36987,"children":36989},{"className":36988},[248,155],[36990],{"type":150,"tag":169,"props":36991,"children":36993},{"className":36992},[248],[36994],{"type":155,"value":29613},{"type":150,"tag":151,"props":36996,"children":36997},{},[36998,37000,37124,37126,37168,37170],{"type":155,"value":36999},"每生成一个 token，光权重就要搬 32 GB。而这一个 token 的计算量是 ",{"type":150,"tag":169,"props":37001,"children":37003},{"className":37002},[177],[37004,37046],{"type":150,"tag":169,"props":37005,"children":37007},{"className":37006},[182],[37008],{"type":150,"tag":185,"props":37009,"children":37010},{"xmlns":187},[37011],{"type":150,"tag":191,"props":37012,"children":37013},{},[37014,37041],{"type":150,"tag":195,"props":37015,"children":37016},{},[37017,37021,37025,37029,37033,37037],{"type":150,"tag":411,"props":37018,"children":37019},{},[37020],{"type":155,"value":433},{"type":150,"tag":417,"props":37022,"children":37023},{},[37024],{"type":155,"value":5554},{"type":150,"tag":411,"props":37026,"children":37027},{},[37028],{"type":155,"value":36853},{"type":150,"tag":217,"props":37030,"children":37031},{},[37032],{"type":155,"value":24918},{"type":150,"tag":417,"props":37034,"children":37035},{},[37036],{"type":155,"value":1045},{"type":150,"tag":411,"props":37038,"children":37039},{},[37040],{"type":155,"value":36879},{"type":150,"tag":223,"props":37042,"children":37043},{"encoding":225},[37044],{"type":155,"value":37045},"2 \\times 16\\text{B} = 32",{"type":150,"tag":169,"props":37047,"children":37049},{"className":37048,"ariaHidden":214},[232],[37050,37076,37111],{"type":150,"tag":169,"props":37051,"children":37053},{"className":37052},[237],[37054,37058,37063,37067,37072],{"type":150,"tag":169,"props":37055,"children":37057},{"className":37056,"style":5617},[242],[],{"type":150,"tag":169,"props":37059,"children":37061},{"className":37060},[248],[37062],{"type":155,"value":433},{"type":150,"tag":169,"props":37064,"children":37066},{"className":37065,"style":5167},[558],[],{"type":150,"tag":169,"props":37068,"children":37070},{"className":37069},[5172],[37071],{"type":155,"value":5554},{"type":150,"tag":169,"props":37073,"children":37075},{"className":37074,"style":5167},[558],[],{"type":150,"tag":169,"props":37077,"children":37079},{"className":37078},[237],[37080,37084,37089,37098,37102,37107],{"type":150,"tag":169,"props":37081,"children":37083},{"className":37082,"style":5854},[242],[],{"type":150,"tag":169,"props":37085,"children":37087},{"className":37086},[248],[37088],{"type":155,"value":36853},{"type":150,"tag":169,"props":37090,"children":37092},{"className":37091},[248,155],[37093],{"type":150,"tag":169,"props":37094,"children":37096},{"className":37095},[248],[37097],{"type":155,"value":24918},{"type":150,"tag":169,"props":37099,"children":37101},{"className":37100,"style":839},[558],[],{"type":150,"tag":169,"props":37103,"children":37105},{"className":37104},[844],[37106],{"type":155,"value":1045},{"type":150,"tag":169,"props":37108,"children":37110},{"className":37109,"style":839},[558],[],{"type":150,"tag":169,"props":37112,"children":37114},{"className":37113},[237],[37115,37119],{"type":150,"tag":169,"props":37116,"children":37118},{"className":37117,"style":5186},[242],[],{"type":150,"tag":169,"props":37120,"children":37122},{"className":37121},[248],[37123],{"type":155,"value":36879},{"type":155,"value":37125}," GFLOPs。算术强度大约是 ",{"type":150,"tag":169,"props":37127,"children":37129},{"className":37128},[177],[37130,37151],{"type":150,"tag":169,"props":37131,"children":37133},{"className":37132},[182],[37134],{"type":150,"tag":185,"props":37135,"children":37136},{"xmlns":187},[37137],{"type":150,"tag":191,"props":37138,"children":37139},{},[37140,37147],{"type":150,"tag":195,"props":37141,"children":37142},{},[37143],{"type":150,"tag":411,"props":37144,"children":37145},{},[37146],{"type":155,"value":415},{"type":150,"tag":223,"props":37148,"children":37149},{"encoding":225},[37150],{"type":155,"value":415},{"type":150,"tag":169,"props":37152,"children":37154},{"className":37153,"ariaHidden":214},[232],[37155],{"type":150,"tag":169,"props":37156,"children":37158},{"className":37157},[237],[37159,37163],{"type":150,"tag":169,"props":37160,"children":37162},{"className":37161,"style":5186},[242],[],{"type":150,"tag":169,"props":37164,"children":37166},{"className":37165},[248],[37167],{"type":155,"value":415},{"type":155,"value":37169}," FLOP\u002Fbyte。现代 GPU 的算力带宽比是几十到几百 FLOP\u002Fbyte。差了将近两个数量级，意味着算力几乎全程在等内存。所以在 decode 这一侧，多算一点几乎不花钱。反正算力在空转等数据。既然这样，就没有理由在生成端省参数量。",{"type":150,"tag":158,"props":37171,"children":37172},{},[37173],{"type":155,"value":37174},"16B 是这么来的。",{"type":150,"tag":151,"props":37176,"children":37177},{},[37178],{"type":155,"value":37179},"（这里按\"每步只生成一个 token\"算，是个简化。speculative decoding 会一次验证多个候选 token，test-time scaling、parallel decoding 这类技术也会改变每步的计算和内存特征。但它们不改变这条主线：无论一步生成几个 token，权重该读的还是要读，KV Cache 该读的还是要读。",{"type":150,"tag":151,"props":37181,"children":37182},{},[37183],{"type":155,"value":37184},"speculative decoding 甚至更吃带宽，草稿模型本身要读一遍权重，验证阶段还要再读主模型。）",{"type":150,"tag":169,"props":37186,"children":37188},{"className":37187},[172],[37189],{"type":150,"tag":169,"props":37190,"children":37192},{"className":37191},[177],[37193,37228],{"type":150,"tag":169,"props":37194,"children":37196},{"className":37195},[182],[37197],{"type":150,"tag":185,"props":37198,"children":37199},{"xmlns":187,"display":188},[37200],{"type":150,"tag":191,"props":37201,"children":37202},{},[37203,37223],{"type":150,"tag":195,"props":37204,"children":37205},{},[37206],{"type":150,"tag":199,"props":37207,"children":37208},{"notation":201},[37209],{"type":150,"tag":204,"props":37210,"children":37211},{"scriptlevel":206,"displaystyle":207},[37212],{"type":150,"tag":204,"props":37213,"children":37214},{"scriptlevel":206,"displaystyle":207},[37215],{"type":150,"tag":204,"props":37216,"children":37217},{"scriptlevel":206,"displaystyle":214},[37218],{"type":150,"tag":217,"props":37219,"children":37220},{},[37221],{"type":155,"value":37222},"Compute bound 的那一侧压计算，memory bound 的那一侧压不住，那就把能力堆上去。",{"type":150,"tag":223,"props":37224,"children":37225},{"encoding":225},[37226],{"type":155,"value":37227},"\\boxed{\\text{Compute bound 的那一侧压计算，memory bound 的那一侧压不住，那就把能力堆上去。}}",{"type":150,"tag":169,"props":37229,"children":37231},{"className":37230,"ariaHidden":214},[232],[37232],{"type":150,"tag":169,"props":37233,"children":37235},{"className":37234},[237],[37236,37240],{"type":150,"tag":169,"props":37237,"children":37239},{"className":37238,"style":30651},[242],[],{"type":150,"tag":169,"props":37241,"children":37243},{"className":37242},[248],[37244],{"type":150,"tag":169,"props":37245,"children":37247},{"className":37246},[253,254],[37248,37319],{"type":150,"tag":169,"props":37249,"children":37251},{"className":37250},[259],[37252,37314],{"type":150,"tag":169,"props":37253,"children":37255},{"className":37254,"style":30668},[264],[37256,37303],{"type":150,"tag":169,"props":37257,"children":37258},{"style":30672},[37259,37263],{"type":150,"tag":169,"props":37260,"children":37262},{"className":37261,"style":30677},[274],[],{"type":150,"tag":169,"props":37264,"children":37266},{"className":37265},[280],[37267],{"type":150,"tag":169,"props":37268,"children":37270},{"className":37269},[248],[37271],{"type":150,"tag":169,"props":37272,"children":37274},{"className":37273},[248],[37275],{"type":150,"tag":169,"props":37276,"children":37278},{"className":37277},[248,155],[37279,37285,37291,37297],{"type":150,"tag":169,"props":37280,"children":37282},{"className":37281},[248],[37283],{"type":155,"value":37284},"Compute bound ",{"type":150,"tag":169,"props":37286,"children":37288},{"className":37287},[248,297],[37289],{"type":155,"value":37290},"的那一侧压计算，",{"type":150,"tag":169,"props":37292,"children":37294},{"className":37293},[248],[37295],{"type":155,"value":37296},"memory bound ",{"type":150,"tag":169,"props":37298,"children":37300},{"className":37299},[248,297],[37301],{"type":155,"value":37302},"的那一侧压不住，那就把能力堆上去。",{"type":150,"tag":169,"props":37304,"children":37305},{"style":30721},[37306,37310],{"type":150,"tag":169,"props":37307,"children":37309},{"className":37308,"style":30677},[274],[],{"type":150,"tag":169,"props":37311,"children":37313},{"className":37312,"style":30730},[348,349],[],{"type":150,"tag":169,"props":37315,"children":37317},{"className":37316},[355],[37318],{"type":155,"value":358},{"type":150,"tag":169,"props":37320,"children":37322},{"className":37321},[259],[37323],{"type":150,"tag":169,"props":37324,"children":37326},{"className":37325,"style":366},[264],[37327],{"type":150,"tag":169,"props":37328,"children":37329},{},[],{"type":150,"tag":151,"props":37331,"children":37332},{},[37333],{"type":155,"value":37334},"（这个读法是我的理解，不是官方给出的设计理由。但 8B 和 16B 正好落在 prefill 和 decode 两侧，而且方向符合各自的瓶颈，我觉得不像是巧合。）传统 decoder-only LLM 对 prompt 里每个 token 和 decode token 用几乎同一套昂贵 backbone，等于强迫两个瓶颈完全不同的阶段共用一套参数。",{"type":150,"tag":151,"props":37336,"children":37337},{},[37338,37340,37430],{"type":155,"value":37339},"如果输入有 ",{"type":150,"tag":169,"props":37341,"children":37343},{"className":37342},[177],[37344,37378],{"type":150,"tag":169,"props":37345,"children":37347},{"className":37346},[182],[37348],{"type":150,"tag":185,"props":37349,"children":37350},{"xmlns":187},[37351],{"type":150,"tag":191,"props":37352,"children":37353},{},[37354,37373],{"type":150,"tag":195,"props":37355,"children":37356},{},[37357,37361,37365,37369],{"type":150,"tag":405,"props":37358,"children":37359},{},[37360],{"type":155,"value":5591},{"type":150,"tag":417,"props":37362,"children":37363},{},[37364],{"type":155,"value":1045},{"type":150,"tag":411,"props":37366,"children":37367},{},[37368],{"type":155,"value":415},{"type":150,"tag":217,"props":37370,"children":37371},{},[37372],{"type":155,"value":6060},{"type":150,"tag":223,"props":37374,"children":37375},{"encoding":225},[37376],{"type":155,"value":37377},"N = 1\\text{M}",{"type":150,"tag":169,"props":37379,"children":37381},{"className":37380,"ariaHidden":214},[232],[37382,37408],{"type":150,"tag":169,"props":37383,"children":37385},{"className":37384},[237],[37386,37390,37395,37399,37404],{"type":150,"tag":169,"props":37387,"children":37389},{"className":37388,"style":5854},[242],[],{"type":150,"tag":169,"props":37391,"children":37393},{"className":37392,"style":5780},[248,489],[37394],{"type":155,"value":5591},{"type":150,"tag":169,"props":37396,"children":37398},{"className":37397,"style":839},[558],[],{"type":150,"tag":169,"props":37400,"children":37402},{"className":37401},[844],[37403],{"type":155,"value":1045},{"type":150,"tag":169,"props":37405,"children":37407},{"className":37406,"style":839},[558],[],{"type":150,"tag":169,"props":37409,"children":37411},{"className":37410},[237],[37412,37416,37421],{"type":150,"tag":169,"props":37413,"children":37415},{"className":37414,"style":5854},[242],[],{"type":150,"tag":169,"props":37417,"children":37419},{"className":37418},[248],[37420],{"type":155,"value":415},{"type":150,"tag":169,"props":37422,"children":37424},{"className":37423},[248,155],[37425],{"type":150,"tag":169,"props":37426,"children":37428},{"className":37427},[248],[37429],{"type":155,"value":6060},{"type":155,"value":37431},"，那么即使 attention 已经 sparse 或 compressed，你依然需要对百万 token 做很多层计算。V4.1 攻击的不只是 attention matrix：",{"type":150,"tag":169,"props":37433,"children":37435},{"className":37434},[172],[37436],{"type":150,"tag":169,"props":37437,"children":37439},{"className":37438},[177],[37440,37475],{"type":150,"tag":169,"props":37441,"children":37443},{"className":37442},[182],[37444],{"type":150,"tag":185,"props":37445,"children":37446},{"xmlns":187,"display":188},[37447],{"type":150,"tag":191,"props":37448,"children":37449},{},[37450,37470],{"type":150,"tag":195,"props":37451,"children":37452},{},[37453,37458],{"type":150,"tag":405,"props":37454,"children":37455},{},[37456],{"type":155,"value":37457},"Q",{"type":150,"tag":746,"props":37459,"children":37460},{},[37461,37466],{"type":150,"tag":405,"props":37462,"children":37463},{},[37464],{"type":155,"value":37465},"K",{"type":150,"tag":405,"props":37467,"children":37468},{"mathvariant":1796},[37469],{"type":155,"value":2052},{"type":150,"tag":223,"props":37471,"children":37472},{"encoding":225},[37473],{"type":155,"value":37474},"QK^\\top",{"type":150,"tag":169,"props":37476,"children":37478},{"className":37477,"ariaHidden":214},[232],[37479],{"type":150,"tag":169,"props":37480,"children":37482},{"className":37481},[237],[37483,37488,37493],{"type":150,"tag":169,"props":37484,"children":37487},{"className":37485,"style":37486},[242],"height:1.0935em;vertical-align:-0.1944em;",[],{"type":150,"tag":169,"props":37489,"children":37491},{"className":37490},[248,489],[37492],{"type":155,"value":37457},{"type":150,"tag":169,"props":37494,"children":37496},{"className":37495},[248],[37497,37502],{"type":150,"tag":169,"props":37498,"children":37500},{"className":37499,"style":6373},[248,489],[37501],{"type":155,"value":37465},{"type":150,"tag":169,"props":37503,"children":37505},{"className":37504},[495],[37506],{"type":150,"tag":169,"props":37507,"children":37509},{"className":37508},[253],[37510],{"type":150,"tag":169,"props":37511,"children":37513},{"className":37512},[259],[37514],{"type":150,"tag":169,"props":37515,"children":37517},{"className":37516,"style":3999},[264],[37518],{"type":150,"tag":169,"props":37519,"children":37520},{"style":3908},[37521,37525],{"type":150,"tag":169,"props":37522,"children":37524},{"className":37523,"style":517},[274],[],{"type":150,"tag":169,"props":37526,"children":37528},{"className":37527},[522,523,524,525],[37529],{"type":150,"tag":169,"props":37530,"children":37532},{"className":37531},[248,525],[37533],{"type":155,"value":2052},{"type":150,"tag":151,"props":37535,"children":37536},{},[37537,37539,37624,37626,37747,37749,37870,37871,37913,37915,38000],{"type":155,"value":37538},"还攻击整个长序列每层 FFN\u002FMoE 的计算。这一点非常重要，因为前面那张账单上的第二项，一直被 ",{"type":150,"tag":169,"props":37540,"children":37542},{"className":37541},[177],[37543,37571],{"type":150,"tag":169,"props":37544,"children":37546},{"className":37545},[182],[37547],{"type":150,"tag":185,"props":37548,"children":37549},{"xmlns":187},[37550],{"type":150,"tag":191,"props":37551,"children":37552},{},[37553,37567],{"type":150,"tag":195,"props":37554,"children":37555},{},[37556],{"type":150,"tag":746,"props":37557,"children":37558},{},[37559,37563],{"type":150,"tag":405,"props":37560,"children":37561},{},[37562],{"type":155,"value":5591},{"type":150,"tag":411,"props":37564,"children":37565},{},[37566],{"type":155,"value":433},{"type":150,"tag":223,"props":37568,"children":37569},{"encoding":225},[37570],{"type":155,"value":7520},{"type":150,"tag":169,"props":37572,"children":37574},{"className":37573,"ariaHidden":214},[232],[37575],{"type":150,"tag":169,"props":37576,"children":37578},{"className":37577},[237],[37579,37583],{"type":150,"tag":169,"props":37580,"children":37582},{"className":37581,"style":7005},[242],[],{"type":150,"tag":169,"props":37584,"children":37586},{"className":37585},[248],[37587,37592],{"type":150,"tag":169,"props":37588,"children":37590},{"className":37589,"style":5780},[248,489],[37591],{"type":155,"value":5591},{"type":150,"tag":169,"props":37593,"children":37595},{"className":37594},[495],[37596],{"type":150,"tag":169,"props":37597,"children":37599},{"className":37598},[253],[37600],{"type":150,"tag":169,"props":37601,"children":37603},{"className":37602},[259],[37604],{"type":150,"tag":169,"props":37605,"children":37607},{"className":37606,"style":7005},[264],[37608],{"type":150,"tag":169,"props":37609,"children":37610},{"style":888},[37611,37615],{"type":150,"tag":169,"props":37612,"children":37614},{"className":37613,"style":517},[274],[],{"type":150,"tag":169,"props":37616,"children":37618},{"className":37617},[522,523,524,525],[37619],{"type":150,"tag":169,"props":37620,"children":37622},{"className":37621},[248,525],[37623],{"type":155,"value":433},{"type":155,"value":37625}," 的光环盖着。attention 那项是 ",{"type":150,"tag":169,"props":37627,"children":37629},{"className":37628},[177],[37630,37674],{"type":150,"tag":169,"props":37631,"children":37633},{"className":37632},[182],[37634],{"type":150,"tag":185,"props":37635,"children":37636},{"xmlns":187},[37637],{"type":150,"tag":191,"props":37638,"children":37639},{},[37640,37670],{"type":150,"tag":195,"props":37641,"children":37642},{},[37643,37647,37651,37662,37666],{"type":150,"tag":405,"props":37644,"children":37645},{},[37646],{"type":155,"value":6923},{"type":150,"tag":417,"props":37648,"children":37649},{"stretchy":207},[37650],{"type":155,"value":972},{"type":150,"tag":746,"props":37652,"children":37653},{},[37654,37658],{"type":150,"tag":405,"props":37655,"children":37656},{},[37657],{"type":155,"value":5591},{"type":150,"tag":411,"props":37659,"children":37660},{},[37661],{"type":155,"value":433},{"type":150,"tag":405,"props":37663,"children":37664},{},[37665],{"type":155,"value":759},{"type":150,"tag":417,"props":37667,"children":37668},{"stretchy":207},[37669],{"type":155,"value":982},{"type":150,"tag":223,"props":37671,"children":37672},{"encoding":225},[37673],{"type":155,"value":7078},{"type":150,"tag":169,"props":37675,"children":37677},{"className":37676,"ariaHidden":214},[232],[37678],{"type":150,"tag":169,"props":37679,"children":37681},{"className":37680},[237],[37682,37686,37691,37696,37737,37742],{"type":150,"tag":169,"props":37683,"children":37685},{"className":37684,"style":6963},[242],[],{"type":150,"tag":169,"props":37687,"children":37689},{"className":37688,"style":6968},[248,489],[37690],{"type":155,"value":6923},{"type":150,"tag":169,"props":37692,"children":37694},{"className":37693},[1004],[37695],{"type":155,"value":972},{"type":150,"tag":169,"props":37697,"children":37699},{"className":37698},[248],[37700,37705],{"type":150,"tag":169,"props":37701,"children":37703},{"className":37702,"style":5780},[248,489],[37704],{"type":155,"value":5591},{"type":150,"tag":169,"props":37706,"children":37708},{"className":37707},[495],[37709],{"type":150,"tag":169,"props":37710,"children":37712},{"className":37711},[253],[37713],{"type":150,"tag":169,"props":37714,"children":37716},{"className":37715},[259],[37717],{"type":150,"tag":169,"props":37718,"children":37720},{"className":37719,"style":7005},[264],[37721],{"type":150,"tag":169,"props":37722,"children":37723},{"style":888},[37724,37728],{"type":150,"tag":169,"props":37725,"children":37727},{"className":37726,"style":517},[274],[],{"type":150,"tag":169,"props":37729,"children":37731},{"className":37730},[522,523,524,525],[37732],{"type":150,"tag":169,"props":37733,"children":37735},{"className":37734},[248,525],[37736],{"type":155,"value":433},{"type":150,"tag":169,"props":37738,"children":37740},{"className":37739},[248,489],[37741],{"type":155,"value":759},{"type":150,"tag":169,"props":37743,"children":37745},{"className":37744},[1015],[37746],{"type":155,"value":982},{"type":155,"value":37748},"，FFN\u002FMoE 那项是 ",{"type":150,"tag":169,"props":37750,"children":37752},{"className":37751},[177],[37753,37797],{"type":150,"tag":169,"props":37754,"children":37756},{"className":37755},[182],[37757],{"type":150,"tag":185,"props":37758,"children":37759},{"xmlns":187},[37760],{"type":150,"tag":191,"props":37761,"children":37762},{},[37763,37793],{"type":150,"tag":195,"props":37764,"children":37765},{},[37766,37770,37774,37778,37789],{"type":150,"tag":405,"props":37767,"children":37768},{},[37769],{"type":155,"value":6923},{"type":150,"tag":417,"props":37771,"children":37772},{"stretchy":207},[37773],{"type":155,"value":972},{"type":150,"tag":405,"props":37775,"children":37776},{},[37777],{"type":155,"value":5591},{"type":150,"tag":746,"props":37779,"children":37780},{},[37781,37785],{"type":150,"tag":405,"props":37782,"children":37783},{},[37784],{"type":155,"value":759},{"type":150,"tag":411,"props":37786,"children":37787},{},[37788],{"type":155,"value":433},{"type":150,"tag":417,"props":37790,"children":37791},{"stretchy":207},[37792],{"type":155,"value":982},{"type":150,"tag":223,"props":37794,"children":37795},{"encoding":225},[37796],{"type":155,"value":6951},{"type":150,"tag":169,"props":37798,"children":37800},{"className":37799,"ariaHidden":214},[232],[37801],{"type":150,"tag":169,"props":37802,"children":37804},{"className":37803},[237],[37805,37809,37814,37819,37824,37865],{"type":150,"tag":169,"props":37806,"children":37808},{"className":37807,"style":6963},[242],[],{"type":150,"tag":169,"props":37810,"children":37812},{"className":37811,"style":6968},[248,489],[37813],{"type":155,"value":6923},{"type":150,"tag":169,"props":37815,"children":37817},{"className":37816},[1004],[37818],{"type":155,"value":972},{"type":150,"tag":169,"props":37820,"children":37822},{"className":37821,"style":5780},[248,489],[37823],{"type":155,"value":5591},{"type":150,"tag":169,"props":37825,"children":37827},{"className":37826},[248],[37828,37833],{"type":150,"tag":169,"props":37829,"children":37831},{"className":37830},[248,489],[37832],{"type":155,"value":759},{"type":150,"tag":169,"props":37834,"children":37836},{"className":37835},[495],[37837],{"type":150,"tag":169,"props":37838,"children":37840},{"className":37839},[253],[37841],{"type":150,"tag":169,"props":37842,"children":37844},{"className":37843},[259],[37845],{"type":150,"tag":169,"props":37846,"children":37848},{"className":37847,"style":7005},[264],[37849],{"type":150,"tag":169,"props":37850,"children":37851},{"style":888},[37852,37856],{"type":150,"tag":169,"props":37853,"children":37855},{"className":37854,"style":517},[274],[],{"type":150,"tag":169,"props":37857,"children":37859},{"className":37858},[522,523,524,525],[37860],{"type":150,"tag":169,"props":37861,"children":37863},{"className":37862},[248,525],[37864],{"type":155,"value":433},{"type":150,"tag":169,"props":37866,"children":37868},{"className":37867},[1015],[37869],{"type":155,"value":982},{"type":155,"value":904},{"type":150,"tag":169,"props":37872,"children":37874},{"className":37873},[177],[37875,37896],{"type":150,"tag":169,"props":37876,"children":37878},{"className":37877},[182],[37879],{"type":150,"tag":185,"props":37880,"children":37881},{"xmlns":187},[37882],{"type":150,"tag":191,"props":37883,"children":37884},{},[37885,37892],{"type":150,"tag":195,"props":37886,"children":37887},{},[37888],{"type":150,"tag":405,"props":37889,"children":37890},{},[37891],{"type":155,"value":5591},{"type":150,"tag":223,"props":37893,"children":37894},{"encoding":225},[37895],{"type":155,"value":5591},{"type":150,"tag":169,"props":37897,"children":37899},{"className":37898,"ariaHidden":214},[232],[37900],{"type":150,"tag":169,"props":37901,"children":37903},{"className":37902},[237],[37904,37908],{"type":150,"tag":169,"props":37905,"children":37907},{"className":37906,"style":5854},[242],[],{"type":150,"tag":169,"props":37909,"children":37911},{"className":37910,"style":5780},[248,489],[37912],{"type":155,"value":5591},{"type":155,"value":37914}," 大到百万量级、",{"type":150,"tag":169,"props":37916,"children":37918},{"className":37917},[177],[37919,37947],{"type":150,"tag":169,"props":37920,"children":37922},{"className":37921},[182],[37923],{"type":150,"tag":185,"props":37924,"children":37925},{"xmlns":187},[37926],{"type":150,"tag":191,"props":37927,"children":37928},{},[37929,37943],{"type":150,"tag":195,"props":37930,"children":37931},{},[37932],{"type":150,"tag":746,"props":37933,"children":37934},{},[37935,37939],{"type":150,"tag":405,"props":37936,"children":37937},{},[37938],{"type":155,"value":5591},{"type":150,"tag":411,"props":37940,"children":37941},{},[37942],{"type":155,"value":433},{"type":150,"tag":223,"props":37944,"children":37945},{"encoding":225},[37946],{"type":155,"value":7520},{"type":150,"tag":169,"props":37948,"children":37950},{"className":37949,"ariaHidden":214},[232],[37951],{"type":150,"tag":169,"props":37952,"children":37954},{"className":37953},[237],[37955,37959],{"type":150,"tag":169,"props":37956,"children":37958},{"className":37957,"style":7005},[242],[],{"type":150,"tag":169,"props":37960,"children":37962},{"className":37961},[248],[37963,37968],{"type":150,"tag":169,"props":37964,"children":37966},{"className":37965,"style":5780},[248,489],[37967],{"type":155,"value":5591},{"type":150,"tag":169,"props":37969,"children":37971},{"className":37970},[495],[37972],{"type":150,"tag":169,"props":37973,"children":37975},{"className":37974},[253],[37976],{"type":150,"tag":169,"props":37977,"children":37979},{"className":37978},[259],[37980],{"type":150,"tag":169,"props":37981,"children":37983},{"className":37982,"style":7005},[264],[37984],{"type":150,"tag":169,"props":37985,"children":37986},{"style":888},[37987,37991],{"type":150,"tag":169,"props":37988,"children":37990},{"className":37989,"style":517},[274],[],{"type":150,"tag":169,"props":37992,"children":37994},{"className":37993},[522,523,524,525],[37995],{"type":150,"tag":169,"props":37996,"children":37998},{"className":37997},[248,525],[37999],{"type":155,"value":433},{"type":155,"value":38001}," 又被打下来之后，后者因为每一层都要对每个 token 算一遍，纹丝不动。",{"type":150,"tag":151,"props":38003,"children":38004},{},[38005],{"type":155,"value":38006},"所以 V4.1 做 asymmetric compute，本质上是：attention 都已经省下来了，总不能剩下的网络继续拿同样预算烧百万 token。终于有人检查了完整账单，而不是只盯着最显眼的那一行。",{"type":150,"tag":151,"props":38008,"children":38009},{},[38010,38012,38019],{"type":155,"value":38011},"省下来的成本是多少？ds4.1f 的技术报告（",{"type":150,"tag":9850,"props":38013,"children":38016},{"href":38014,"rel":38015},"https:\u002F\u002Fwww.alphaxiv.org\u002Fabs\u002F2609.deepseek-v4-1-flash",[9854],[38017],{"type":155,"value":38018},"DeepSeek-V4.1-Flash: Pushing the Limits of KV Cache Compression",{"type":155,"value":38020},"）给了具体的数。相比上一代 V4-Flash：",{"type":150,"tag":23833,"props":38022,"children":38023},{},[38024,38048],{"type":150,"tag":23837,"props":38025,"children":38026},{},[38027,38029,38034,38036,38041,38043],{"type":155,"value":38028},"常驻 ",{"type":150,"tag":158,"props":38030,"children":38031},{},[38032],{"type":155,"value":38033},"HBM",{"type":155,"value":38035}," 的 global KV cache：",{"type":150,"tag":158,"props":38037,"children":38038},{},[38039],{"type":155,"value":38040},"890 字节 \u002F token",{"type":155,"value":38042},"，约为原来的 ",{"type":150,"tag":158,"props":38044,"children":38045},{},[38046],{"type":155,"value":38047},"1\u002F4",{"type":150,"tag":23837,"props":38049,"children":38050},{},[38051,38053,38058,38060],{"type":155,"value":38052},"落在 ",{"type":150,"tag":158,"props":38054,"children":38055},{},[38056],{"type":155,"value":38057},"SSD 或 host memory",{"type":155,"value":38059}," 的 persistent KV cache：约为原来的 ",{"type":150,"tag":158,"props":38061,"children":38062},{},[38063],{"type":155,"value":38064},"1\u002F8",{"type":150,"tag":151,"props":38066,"children":38067},{},[38068,38070,38112,38114,38156,38158,38200],{"type":155,"value":38069},"报告把这些收益归因于\"entry size、sequence dimension、layer dimension 三个方向上的协同优化\"，落到 CSA2 上就是三个模式：投影让 head 之间共享信息（压 ",{"type":150,"tag":169,"props":38071,"children":38073},{"className":38072},[177],[38074,38095],{"type":150,"tag":169,"props":38075,"children":38077},{"className":38076},[182],[38078],{"type":150,"tag":185,"props":38079,"children":38080},{"xmlns":187},[38081],{"type":150,"tag":191,"props":38082,"children":38083},{},[38084,38091],{"type":150,"tag":195,"props":38085,"children":38086},{},[38087],{"type":150,"tag":405,"props":38088,"children":38089},{},[38090],{"type":155,"value":35647},{"type":150,"tag":223,"props":38092,"children":38093},{"encoding":225},[38094],{"type":155,"value":35647},{"type":150,"tag":169,"props":38096,"children":38098},{"className":38097,"ariaHidden":214},[232],[38099],{"type":150,"tag":169,"props":38100,"children":38102},{"className":38101},[237],[38103,38107],{"type":150,"tag":169,"props":38104,"children":38106},{"className":38105,"style":5854},[242],[],{"type":150,"tag":169,"props":38108,"children":38110},{"className":38109,"style":6968},[248,489],[38111],{"type":155,"value":35647},{"type":155,"value":38113},"）、把多个 token 合成一个 KV entry（压 ",{"type":150,"tag":169,"props":38115,"children":38117},{"className":38116},[177],[38118,38139],{"type":150,"tag":169,"props":38119,"children":38121},{"className":38120},[182],[38122],{"type":150,"tag":185,"props":38123,"children":38124},{"xmlns":187},[38125],{"type":150,"tag":191,"props":38126,"children":38127},{},[38128,38135],{"type":150,"tag":195,"props":38129,"children":38130},{},[38131],{"type":150,"tag":405,"props":38132,"children":38133},{},[38134],{"type":155,"value":5591},{"type":150,"tag":223,"props":38136,"children":38137},{"encoding":225},[38138],{"type":155,"value":5591},{"type":150,"tag":169,"props":38140,"children":38142},{"className":38141,"ariaHidden":214},[232],[38143],{"type":150,"tag":169,"props":38144,"children":38146},{"className":38145},[237],[38147,38151],{"type":150,"tag":169,"props":38148,"children":38150},{"className":38149,"style":5854},[242],[],{"type":150,"tag":169,"props":38152,"children":38154},{"className":38153,"style":5780},[248,489],[38155],{"type":155,"value":5591},{"type":155,"value":38157},"）、跨层复用 KV（压 ",{"type":150,"tag":169,"props":38159,"children":38161},{"className":38160},[177],[38162,38183],{"type":150,"tag":169,"props":38163,"children":38165},{"className":38164},[182],[38166],{"type":150,"tag":185,"props":38167,"children":38168},{"xmlns":187},[38169],{"type":150,"tag":191,"props":38170,"children":38171},{},[38172,38179],{"type":150,"tag":195,"props":38173,"children":38174},{},[38175],{"type":150,"tag":405,"props":38176,"children":38177},{},[38178],{"type":155,"value":5559},{"type":150,"tag":223,"props":38180,"children":38181},{"encoding":225},[38182],{"type":155,"value":5559},{"type":150,"tag":169,"props":38184,"children":38186},{"className":38185,"ariaHidden":214},[232],[38187],{"type":150,"tag":169,"props":38188,"children":38190},{"className":38189},[237],[38191,38195],{"type":150,"tag":169,"props":38192,"children":38194},{"className":38193,"style":5854},[242],[],{"type":150,"tag":169,"props":38196,"children":38198},{"className":38197},[248,489],[38199],{"type":155,"value":5559},{"type":155,"value":38201},"）。再加上主 KV cache 本身换成 FP4。",{"type":150,"tag":151,"props":38203,"children":38204},{},[38205,38322],{"type":150,"tag":158,"props":38206,"children":38207},{},[38208,38210,38321],{"type":155,"value":38209},"这不就是前面的 ",{"type":150,"tag":169,"props":38211,"children":38213},{"className":38212},[177],[38214,38252],{"type":150,"tag":169,"props":38215,"children":38217},{"className":38216},[182],[38218],{"type":150,"tag":185,"props":38219,"children":38220},{"xmlns":187},[38221],{"type":150,"tag":191,"props":38222,"children":38223},{},[38224,38247],{"type":150,"tag":195,"props":38225,"children":38226},{},[38227,38231,38235,38239,38243],{"type":150,"tag":405,"props":38228,"children":38229},{},[38230],{"type":155,"value":5559},{"type":150,"tag":417,"props":38232,"children":38233},{},[38234],{"type":155,"value":5554},{"type":150,"tag":405,"props":38236,"children":38237},{},[38238],{"type":155,"value":5591},{"type":150,"tag":417,"props":38240,"children":38241},{},[38242],{"type":155,"value":5554},{"type":150,"tag":405,"props":38244,"children":38245},{},[38246],{"type":155,"value":35647},{"type":150,"tag":223,"props":38248,"children":38249},{"encoding":225},[38250],{"type":155,"value":38251},"L \\times N \\times D",{"type":150,"tag":169,"props":38253,"children":38255},{"className":38254,"ariaHidden":214},[232],[38256,38282,38308],{"type":150,"tag":169,"props":38257,"children":38259},{"className":38258},[237],[38260,38264,38269,38273,38278],{"type":150,"tag":169,"props":38261,"children":38263},{"className":38262,"style":5644},[242],[],{"type":150,"tag":169,"props":38265,"children":38267},{"className":38266},[248,489],[38268],{"type":155,"value":5559},{"type":150,"tag":169,"props":38270,"children":38272},{"className":38271,"style":5167},[558],[],{"type":150,"tag":169,"props":38274,"children":38276},{"className":38275},[5172],[38277],{"type":155,"value":5554},{"type":150,"tag":169,"props":38279,"children":38281},{"className":38280,"style":5167},[558],[],{"type":150,"tag":169,"props":38283,"children":38285},{"className":38284},[237],[38286,38290,38295,38299,38304],{"type":150,"tag":169,"props":38287,"children":38289},{"className":38288,"style":5644},[242],[],{"type":150,"tag":169,"props":38291,"children":38293},{"className":38292,"style":5780},[248,489],[38294],{"type":155,"value":5591},{"type":150,"tag":169,"props":38296,"children":38298},{"className":38297,"style":5167},[558],[],{"type":150,"tag":169,"props":38300,"children":38302},{"className":38301},[5172],[38303],{"type":155,"value":5554},{"type":150,"tag":169,"props":38305,"children":38307},{"className":38306,"style":5167},[558],[],{"type":150,"tag":169,"props":38309,"children":38311},{"className":38310},[237],[38312,38316],{"type":150,"tag":169,"props":38313,"children":38315},{"className":38314,"style":5854},[242],[],{"type":150,"tag":169,"props":38317,"children":38319},{"className":38318,"style":6968},[248,489],[38320],{"type":155,"value":35647},{"type":155,"value":904},{"type":155,"value":38323}," 前面那条\"压宽度 → 压长度 → 压深度\"的路线，在 ds4.1f 上是第一次三个方向同时动刀。",{"type":150,"tag":151,"props":38325,"children":38326},{},[38327,38329,38371],{"type":155,"value":38328},"压 ",{"type":150,"tag":169,"props":38330,"children":38332},{"className":38331},[177],[38333,38354],{"type":150,"tag":169,"props":38334,"children":38336},{"className":38335},[182],[38337],{"type":150,"tag":185,"props":38338,"children":38339},{"xmlns":187},[38340],{"type":150,"tag":191,"props":38341,"children":38342},{},[38343,38350],{"type":150,"tag":195,"props":38344,"children":38345},{},[38346],{"type":150,"tag":405,"props":38347,"children":38348},{},[38349],{"type":155,"value":5559},{"type":150,"tag":223,"props":38351,"children":38352},{"encoding":225},[38353],{"type":155,"value":5559},{"type":150,"tag":169,"props":38355,"children":38357},{"className":38356,"ariaHidden":214},[232],[38358],{"type":150,"tag":169,"props":38359,"children":38361},{"className":38360},[237],[38362,38366],{"type":150,"tag":169,"props":38363,"children":38365},{"className":38364,"style":5854},[242],[],{"type":150,"tag":169,"props":38367,"children":38369},{"className":38368},[248,489],[38370],{"type":155,"value":5559},{"type":155,"value":38372}," 这一个方向前面一直只是断言，说清楚它得看 CSA2 的三种模式：",{"type":150,"tag":23833,"props":38374,"children":38375},{},[38376,38386,38403],{"type":150,"tag":23837,"props":38377,"children":38378},{},[38379,38384],{"type":150,"tag":158,"props":38380,"children":38381},{},[38382],{"type":155,"value":38383},"Full",{"type":155,"value":38385},"：这层自己算主 K、V，投影一个 indexer key，产出一组新的 Top-K 索引",{"type":150,"tag":23837,"props":38387,"children":38388},{},[38389,38394,38396,38401],{"type":150,"tag":158,"props":38390,"children":38391},{},[38392],{"type":155,"value":38393},"Reindex",{"type":155,"value":38395},"：主 K、V 和 indexer key 都从前一层继承，只重算 indexer query，于是选出一组",{"type":150,"tag":158,"props":38397,"children":38398},{},[38399],{"type":155,"value":38400},"不同",{"type":155,"value":38402},"的 Top-K",{"type":150,"tag":23837,"props":38404,"children":38405},{},[38406,38411],{"type":150,"tag":158,"props":38407,"children":38408},{},[38409],{"type":155,"value":38410},"Reuse",{"type":155,"value":38412},"：主 K、V 和 Top-K 索引全部继承，这层几乎不做索引计算",{"type":150,"tag":151,"props":38414,"children":38415},{},[38416],{"type":155,"value":38417},"再配一个分层的 indexer：第一个 Full 层先圈出一个 block 级的候选池，后面的 Reindex 层只在这个池子里打分。于是每个 query 的索引开销被限住，context 是一万还是一百万都一样。",{"type":150,"tag":151,"props":38419,"children":38420},{},[38421,38423],{"type":155,"value":38422},"这才是\"每一层不再重复为同一份历史付费\"的具体做法。",{"type":150,"tag":158,"props":38424,"children":38425},{},[38426],{"type":155,"value":38427},"不是每层都重建索引，而是让多数层复用、少数层重建。",{"type":150,"tag":151,"props":38429,"children":38430},{},[38431],{"type":155,"value":38432},"持久层那 1\u002F8 靠的是另一个部署优化，叫 SWA Bounded Replay：局部窗口的 cache 不再常驻，请求恢复时只重放最近一小段就能近似重建，不必把整条序列重算一遍。",{"type":150,"tag":151,"props":38434,"children":38435},{},[38436,38438,38542,38544,38984],{"type":155,"value":38437},"另外 CED 让 prefill 的复杂度从 ",{"type":150,"tag":169,"props":38439,"children":38441},{"className":38440},[177],[38442,38484],{"type":150,"tag":169,"props":38443,"children":38445},{"className":38444},[182],[38446],{"type":150,"tag":185,"props":38447,"children":38448},{"xmlns":187},[38449],{"type":150,"tag":191,"props":38450,"children":38451},{},[38452,38479],{"type":150,"tag":195,"props":38453,"children":38454},{},[38455,38459,38463,38467,38471,38475],{"type":150,"tag":405,"props":38456,"children":38457},{},[38458],{"type":155,"value":6923},{"type":150,"tag":417,"props":38460,"children":38461},{"stretchy":207},[38462],{"type":155,"value":972},{"type":150,"tag":405,"props":38464,"children":38465},{},[38466],{"type":155,"value":5591},{"type":150,"tag":417,"props":38468,"children":38469},{},[38470],{"type":155,"value":25192},{"type":150,"tag":405,"props":38472,"children":38473},{},[38474],{"type":155,"value":5559},{"type":150,"tag":417,"props":38476,"children":38477},{"stretchy":207},[38478],{"type":155,"value":982},{"type":150,"tag":223,"props":38480,"children":38481},{"encoding":225},[38482],{"type":155,"value":38483},"O(N \\cdot L)",{"type":150,"tag":169,"props":38485,"children":38487},{"className":38486,"ariaHidden":214},[232],[38488,38524],{"type":150,"tag":169,"props":38489,"children":38491},{"className":38490},[237],[38492,38496,38501,38506,38511,38515,38520],{"type":150,"tag":169,"props":38493,"children":38495},{"className":38494,"style":999},[242],[],{"type":150,"tag":169,"props":38497,"children":38499},{"className":38498,"style":6968},[248,489],[38500],{"type":155,"value":6923},{"type":150,"tag":169,"props":38502,"children":38504},{"className":38503},[1004],[38505],{"type":155,"value":972},{"type":150,"tag":169,"props":38507,"children":38509},{"className":38508,"style":5780},[248,489],[38510],{"type":155,"value":5591},{"type":150,"tag":169,"props":38512,"children":38514},{"className":38513,"style":5167},[558],[],{"type":150,"tag":169,"props":38516,"children":38518},{"className":38517},[5172],[38519],{"type":155,"value":25192},{"type":150,"tag":169,"props":38521,"children":38523},{"className":38522,"style":5167},[558],[],{"type":150,"tag":169,"props":38525,"children":38527},{"className":38526},[237],[38528,38532,38537],{"type":150,"tag":169,"props":38529,"children":38531},{"className":38530,"style":999},[242],[],{"type":150,"tag":169,"props":38533,"children":38535},{"className":38534},[248,489],[38536],{"type":155,"value":5559},{"type":150,"tag":169,"props":38538,"children":38540},{"className":38539},[1015],[38541],{"type":155,"value":982},{"type":155,"value":38543}," 降到大约 ",{"type":150,"tag":169,"props":38545,"children":38547},{"className":38546},[177],[38548,38628],{"type":150,"tag":169,"props":38549,"children":38551},{"className":38550},[182],[38552],{"type":150,"tag":185,"props":38553,"children":38554},{"xmlns":187},[38555],{"type":150,"tag":191,"props":38556,"children":38557},{},[38558,38623],{"type":150,"tag":195,"props":38559,"children":38560},{},[38561,38565,38569,38573,38577,38588,38592,38604,38608,38619],{"type":150,"tag":405,"props":38562,"children":38563},{},[38564],{"type":155,"value":6923},{"type":150,"tag":417,"props":38566,"children":38567},{"stretchy":207},[38568],{"type":155,"value":972},{"type":150,"tag":405,"props":38570,"children":38571},{},[38572],{"type":155,"value":5591},{"type":150,"tag":417,"props":38574,"children":38575},{},[38576],{"type":155,"value":25192},{"type":150,"tag":1952,"props":38578,"children":38579},{},[38580,38584],{"type":150,"tag":405,"props":38581,"children":38582},{},[38583],{"type":155,"value":5559},{"type":150,"tag":411,"props":38585,"children":38586},{},[38587],{"type":155,"value":433},{"type":150,"tag":417,"props":38589,"children":38590},{},[38591],{"type":155,"value":5136},{"type":150,"tag":401,"props":38593,"children":38594},{},[38595,38599],{"type":150,"tag":405,"props":38596,"children":38597},{},[38598],{"type":155,"value":1817},{"type":150,"tag":217,"props":38600,"children":38601},{},[38602],{"type":155,"value":38603},"win",{"type":150,"tag":417,"props":38605,"children":38606},{},[38607],{"type":155,"value":25192},{"type":150,"tag":1952,"props":38609,"children":38610},{},[38611,38615],{"type":150,"tag":405,"props":38612,"children":38613},{},[38614],{"type":155,"value":5559},{"type":150,"tag":411,"props":38616,"children":38617},{},[38618],{"type":155,"value":433},{"type":150,"tag":417,"props":38620,"children":38621},{"stretchy":207},[38622],{"type":155,"value":982},{"type":150,"tag":223,"props":38624,"children":38625},{"encoding":225},[38626],{"type":155,"value":38627},"O(N \\cdot \\frac{L}{2} + n_{\\text{win}} \\cdot \\frac{L}{2})",{"type":150,"tag":169,"props":38629,"children":38631},{"className":38630,"ariaHidden":214},[232],[38632,38668,38789,38876],{"type":150,"tag":169,"props":38633,"children":38635},{"className":38634},[237],[38636,38640,38645,38650,38655,38659,38664],{"type":150,"tag":169,"props":38637,"children":38639},{"className":38638,"style":999},[242],[],{"type":150,"tag":169,"props":38641,"children":38643},{"className":38642,"style":6968},[248,489],[38644],{"type":155,"value":6923},{"type":150,"tag":169,"props":38646,"children":38648},{"className":38647},[1004],[38649],{"type":155,"value":972},{"type":150,"tag":169,"props":38651,"children":38653},{"className":38652,"style":5780},[248,489],[38654],{"type":155,"value":5591},{"type":150,"tag":169,"props":38656,"children":38658},{"className":38657,"style":5167},[558],[],{"type":150,"tag":169,"props":38660,"children":38662},{"className":38661},[5172],[38663],{"type":155,"value":25192},{"type":150,"tag":169,"props":38665,"children":38667},{"className":38666,"style":5167},[558],[],{"type":150,"tag":169,"props":38669,"children":38671},{"className":38670},[237],[38672,38677,38776,38780,38785],{"type":150,"tag":169,"props":38673,"children":38676},{"className":38674,"style":38675},[242],"height:1.2173em;vertical-align:-0.345em;",[],{"type":150,"tag":169,"props":38678,"children":38680},{"className":38679},[248],[38681,38685,38772],{"type":150,"tag":169,"props":38682,"children":38684},{"className":38683},[1004,2660],[],{"type":150,"tag":169,"props":38686,"children":38688},{"className":38687},[1952],[38689],{"type":150,"tag":169,"props":38690,"children":38692},{"className":38691},[253,254],[38693,38760],{"type":150,"tag":169,"props":38694,"children":38696},{"className":38695},[259],[38697,38755],{"type":150,"tag":169,"props":38698,"children":38701},{"className":38699,"style":38700},[264],"height:0.8723em;",[38702,38723,38734],{"type":150,"tag":169,"props":38703,"children":38705},{"style":38704},"top:-2.655em;",[38706,38710],{"type":150,"tag":169,"props":38707,"children":38709},{"className":38708,"style":8427},[274],[],{"type":150,"tag":169,"props":38711,"children":38713},{"className":38712},[522,523,524,525],[38714],{"type":150,"tag":169,"props":38715,"children":38717},{"className":38716},[248,525],[38718],{"type":150,"tag":169,"props":38719,"children":38721},{"className":38720},[248,525],[38722],{"type":155,"value":433},{"type":150,"tag":169,"props":38724,"children":38725},{"style":8440},[38726,38730],{"type":150,"tag":169,"props":38727,"children":38729},{"className":38728,"style":8427},[274],[],{"type":150,"tag":169,"props":38731,"children":38733},{"className":38732,"style":2982},[2981],[],{"type":150,"tag":169,"props":38735,"children":38737},{"style":38736},"top:-3.394em;",[38738,38742],{"type":150,"tag":169,"props":38739,"children":38741},{"className":38740,"style":8427},[274],[],{"type":150,"tag":169,"props":38743,"children":38745},{"className":38744},[522,523,524,525],[38746],{"type":150,"tag":169,"props":38747,"children":38749},{"className":38748},[248,525],[38750],{"type":150,"tag":169,"props":38751,"children":38753},{"className":38752},[248,489,525],[38754],{"type":155,"value":5559},{"type":150,"tag":169,"props":38756,"children":38758},{"className":38757},[355],[38759],{"type":155,"value":358},{"type":150,"tag":169,"props":38761,"children":38763},{"className":38762},[259],[38764],{"type":150,"tag":169,"props":38765,"children":38768},{"className":38766,"style":38767},[264],"height:0.345em;",[38769],{"type":150,"tag":169,"props":38770,"children":38771},{},[],{"type":150,"tag":169,"props":38773,"children":38775},{"className":38774},[1015,2660],[],{"type":150,"tag":169,"props":38777,"children":38779},{"className":38778,"style":5167},[558],[],{"type":150,"tag":169,"props":38781,"children":38783},{"className":38782},[5172],[38784],{"type":155,"value":5136},{"type":150,"tag":169,"props":38786,"children":38788},{"className":38787,"style":5167},[558],[],{"type":150,"tag":169,"props":38790,"children":38792},{"className":38791},[237],[38793,38798,38863,38867,38872],{"type":150,"tag":169,"props":38794,"children":38797},{"className":38795,"style":38796},[242],"height:0.5945em;vertical-align:-0.15em;",[],{"type":150,"tag":169,"props":38799,"children":38801},{"className":38800},[248],[38802,38807],{"type":150,"tag":169,"props":38803,"children":38805},{"className":38804},[248,489],[38806],{"type":155,"value":1817},{"type":150,"tag":169,"props":38808,"children":38810},{"className":38809},[495],[38811],{"type":150,"tag":169,"props":38812,"children":38814},{"className":38813},[253,254],[38815,38852],{"type":150,"tag":169,"props":38816,"children":38818},{"className":38817},[259],[38819,38847],{"type":150,"tag":169,"props":38820,"children":38822},{"className":38821,"style":6478},[264],[38823],{"type":150,"tag":169,"props":38824,"children":38825},{"style":512},[38826,38830],{"type":150,"tag":169,"props":38827,"children":38829},{"className":38828,"style":517},[274],[],{"type":150,"tag":169,"props":38831,"children":38833},{"className":38832},[522,523,524,525],[38834],{"type":150,"tag":169,"props":38835,"children":38837},{"className":38836},[248,525],[38838],{"type":150,"tag":169,"props":38839,"children":38841},{"className":38840},[248,155,525],[38842],{"type":150,"tag":169,"props":38843,"children":38845},{"className":38844},[248,525],[38846],{"type":155,"value":38603},{"type":150,"tag":169,"props":38848,"children":38850},{"className":38849},[355],[38851],{"type":155,"value":358},{"type":150,"tag":169,"props":38853,"children":38855},{"className":38854},[259],[38856],{"type":150,"tag":169,"props":38857,"children":38859},{"className":38858,"style":544},[264],[38860],{"type":150,"tag":169,"props":38861,"children":38862},{},[],{"type":150,"tag":169,"props":38864,"children":38866},{"className":38865,"style":5167},[558],[],{"type":150,"tag":169,"props":38868,"children":38870},{"className":38869},[5172],[38871],{"type":155,"value":25192},{"type":150,"tag":169,"props":38873,"children":38875},{"className":38874,"style":5167},[558],[],{"type":150,"tag":169,"props":38877,"children":38879},{"className":38878},[237],[38880,38884,38979],{"type":150,"tag":169,"props":38881,"children":38883},{"className":38882,"style":38675},[242],[],{"type":150,"tag":169,"props":38885,"children":38887},{"className":38886},[248],[38888,38892,38975],{"type":150,"tag":169,"props":38889,"children":38891},{"className":38890},[1004,2660],[],{"type":150,"tag":169,"props":38893,"children":38895},{"className":38894},[1952],[38896],{"type":150,"tag":169,"props":38897,"children":38899},{"className":38898},[253,254],[38900,38964],{"type":150,"tag":169,"props":38901,"children":38903},{"className":38902},[259],[38904,38959],{"type":150,"tag":169,"props":38905,"children":38907},{"className":38906,"style":38700},[264],[38908,38928,38939],{"type":150,"tag":169,"props":38909,"children":38910},{"style":38704},[38911,38915],{"type":150,"tag":169,"props":38912,"children":38914},{"className":38913,"style":8427},[274],[],{"type":150,"tag":169,"props":38916,"children":38918},{"className":38917},[522,523,524,525],[38919],{"type":150,"tag":169,"props":38920,"children":38922},{"className":38921},[248,525],[38923],{"type":150,"tag":169,"props":38924,"children":38926},{"className":38925},[248,525],[38927],{"type":155,"value":433},{"type":150,"tag":169,"props":38929,"children":38930},{"style":8440},[38931,38935],{"type":150,"tag":169,"props":38932,"children":38934},{"className":38933,"style":8427},[274],[],{"type":150,"tag":169,"props":38936,"children":38938},{"className":38937,"style":2982},[2981],[],{"type":150,"tag":169,"props":38940,"children":38941},{"style":38736},[38942,38946],{"type":150,"tag":169,"props":38943,"children":38945},{"className":38944,"style":8427},[274],[],{"type":150,"tag":169,"props":38947,"children":38949},{"className":38948},[522,523,524,525],[38950],{"type":150,"tag":169,"props":38951,"children":38953},{"className":38952},[248,525],[38954],{"type":150,"tag":169,"props":38955,"children":38957},{"className":38956},[248,489,525],[38958],{"type":155,"value":5559},{"type":150,"tag":169,"props":38960,"children":38962},{"className":38961},[355],[38963],{"type":155,"value":358},{"type":150,"tag":169,"props":38965,"children":38967},{"className":38966},[259],[38968],{"type":150,"tag":169,"props":38969,"children":38971},{"className":38970,"style":38767},[264],[38972],{"type":150,"tag":169,"props":38973,"children":38974},{},[],{"type":150,"tag":169,"props":38976,"children":38978},{"className":38977},[1015,2660],[],{"type":150,"tag":169,"props":38980,"children":38982},{"className":38981},[1015],[38983],{"type":155,"value":982},{"type":155,"value":38985},"，长输入的计算量差不多减半。",{"type":150,"tag":151,"props":38987,"children":38988},{},[38989,38991,38998],{"type":155,"value":38990},"那 890 字节 \u002F token 在真实硬件上意味着什么。拿 B200 算一遍（",{"type":150,"tag":9850,"props":38992,"children":38995},{"href":38993,"rel":38994},"https:\u002F\u002Finferencex.semianalysis.com\u002Fchips\u002Fb200",[9854],[38996],{"type":155,"value":38997},"规格",{"type":155,"value":38999},"）。",{"type":150,"tag":151,"props":39001,"children":39002},{},[39003,39005,39010],{"type":155,"value":39004},"先说部署形态。552B 权重按 FP4 存是 276 GB，单卡 180 GB 装不下，所以要拆卡。",{"type":150,"tag":158,"props":39006,"children":39007},{},[39008],{"type":155,"value":39009},"四卡（TP4）就够",{"type":155,"value":39011},"：每卡 69 GB 权重，还剩 111 GB 放 KV。",{"type":150,"tag":151,"props":39013,"children":39014},{},[39015,39017,39022],{"type":155,"value":39016},"下面算的都是",{"type":150,"tag":158,"props":39018,"children":39019},{},[39020],{"type":155,"value":39021},"每张卡",{"type":155,"value":39023},"的数字。TP4 之后每卡只承担四分之一，所以字面值和耗时都除以 4，算术强度不变。",{"type":150,"tag":151,"props":39025,"children":39026},{},[39027],{"type":155,"value":39028},"先把精度定死，因为算力和字节数都随它变：",{"type":150,"tag":22368,"props":39030,"children":39031},{},[39032,39051],{"type":150,"tag":22372,"props":39033,"children":39034},{},[39035],{"type":150,"tag":22376,"props":39036,"children":39037},{},[39038,39041,39046],{"type":150,"tag":22380,"props":39039,"children":39040},{},[],{"type":150,"tag":22380,"props":39042,"children":39043},{},[39044],{"type":155,"value":39045},"用什么精度",{"type":150,"tag":22380,"props":39047,"children":39048},{},[39049],{"type":155,"value":39050},"依据",{"type":150,"tag":22394,"props":39052,"children":39053},{},[39054,39071,39089],{"type":150,"tag":22376,"props":39055,"children":39056},{},[39057,39061,39066],{"type":150,"tag":22401,"props":39058,"children":39059},{},[39060],{"type":155,"value":6246},{"type":150,"tag":22401,"props":39062,"children":39063},{},[39064],{"type":155,"value":39065},"FP4",{"type":150,"tag":22401,"props":39067,"children":39068},{},[39069],{"type":155,"value":39070},"每参数 0.5 字节",{"type":150,"tag":22376,"props":39072,"children":39073},{},[39074,39079,39084],{"type":150,"tag":22401,"props":39075,"children":39076},{},[39077],{"type":155,"value":39078},"KV cache",{"type":150,"tag":22401,"props":39080,"children":39081},{},[39082],{"type":155,"value":39083},"FP4（MXFP4）",{"type":150,"tag":22401,"props":39085,"children":39086},{},[39087],{"type":155,"value":39088},"技术报告的 QAT 就是按 MXFP4 做的，890 字节 \u002F token 是这个精度下的值",{"type":150,"tag":22376,"props":39090,"children":39091},{},[39092,39097,39101],{"type":150,"tag":22401,"props":39093,"children":39094},{},[39095],{"type":155,"value":39096},"算力基准",{"type":150,"tag":22401,"props":39098,"children":39099},{},[39100],{"type":155,"value":39065},{"type":150,"tag":22401,"props":39102,"children":39103},{},[39104],{"type":155,"value":39105},"B200 dense 9000 TFLOP\u002Fs",{"type":150,"tag":151,"props":39107,"children":39108},{},[39109],{"type":155,"value":39110},"三项同一把尺子，拐点和字节数才可比。",{"type":150,"tag":151,"props":39112,"children":39113},{},[39114],{"type":155,"value":39115},"B200 的 BF16 \u002F FP8 \u002F FP4 dense 算力分别是 2250 \u002F 4500 \u002F 9000 TFLOP\u002Fs，HBM3e 带宽 8 TB\u002Fs。FP4 的拐点是：",{"type":150,"tag":169,"props":39117,"children":39119},{"className":39118},[172],[39120],{"type":150,"tag":169,"props":39121,"children":39123},{"className":39122},[177],[39124,39181],{"type":150,"tag":169,"props":39125,"children":39127},{"className":39126},[182],[39128],{"type":150,"tag":185,"props":39129,"children":39130},{"xmlns":187,"display":188},[39131],{"type":150,"tag":191,"props":39132,"children":39133},{},[39134,39176],{"type":150,"tag":195,"props":39135,"children":39136},{},[39137,39163,39167,39172],{"type":150,"tag":1952,"props":39138,"children":39139},{},[39140,39152],{"type":150,"tag":195,"props":39141,"children":39142},{},[39143,39148],{"type":150,"tag":411,"props":39144,"children":39145},{},[39146],{"type":155,"value":39147},"9000",{"type":150,"tag":217,"props":39149,"children":39150},{},[39151],{"type":155,"value":23616},{"type":150,"tag":195,"props":39153,"children":39154},{},[39155,39159],{"type":150,"tag":411,"props":39156,"children":39157},{},[39158],{"type":155,"value":9389},{"type":150,"tag":217,"props":39160,"children":39161},{},[39162],{"type":155,"value":23629},{"type":150,"tag":417,"props":39164,"children":39165},{},[39166],{"type":155,"value":1045},{"type":150,"tag":411,"props":39168,"children":39169},{},[39170],{"type":155,"value":39171},"1125",{"type":150,"tag":217,"props":39173,"children":39174},{},[39175],{"type":155,"value":23643},{"type":150,"tag":223,"props":39177,"children":39178},{"encoding":225},[39179],{"type":155,"value":39180},"\\frac{9000\\ \\text{TFLOPS}}{8\\ \\text{TB\u002Fs}} = 1125\\ \\text{FLOP\u002Fbyte}",{"type":150,"tag":169,"props":39182,"children":39184},{"className":39183,"ariaHidden":214},[232],[39185,39321],{"type":150,"tag":169,"props":39186,"children":39188},{"className":39187},[237],[39189,39193,39308,39312,39317],{"type":150,"tag":169,"props":39190,"children":39192},{"className":39191,"style":23660},[242],[],{"type":150,"tag":169,"props":39194,"children":39196},{"className":39195},[248],[39197,39201,39304],{"type":150,"tag":169,"props":39198,"children":39200},{"className":39199},[1004,2660],[],{"type":150,"tag":169,"props":39202,"children":39204},{"className":39203},[1952],[39205],{"type":150,"tag":169,"props":39206,"children":39208},{"className":39207},[253,254],[39209,39293],{"type":150,"tag":169,"props":39210,"children":39212},{"className":39211},[259],[39213,39288],{"type":150,"tag":169,"props":39214,"children":39216},{"className":39215,"style":23685},[264],[39217,39247,39258],{"type":150,"tag":169,"props":39218,"children":39219},{"style":8422},[39220,39224],{"type":150,"tag":169,"props":39221,"children":39223},{"className":39222,"style":8427},[274],[],{"type":150,"tag":169,"props":39225,"children":39227},{"className":39226},[248],[39228,39233,39238],{"type":150,"tag":169,"props":39229,"children":39231},{"className":39230},[248],[39232],{"type":155,"value":9389},{"type":150,"tag":169,"props":39234,"children":39236},{"className":39235},[558],[39237],{"type":155,"value":8763},{"type":150,"tag":169,"props":39239,"children":39241},{"className":39240},[248,155],[39242],{"type":150,"tag":169,"props":39243,"children":39245},{"className":39244},[248],[39246],{"type":155,"value":23717},{"type":150,"tag":169,"props":39248,"children":39249},{"style":8440},[39250,39254],{"type":150,"tag":169,"props":39251,"children":39253},{"className":39252,"style":8427},[274],[],{"type":150,"tag":169,"props":39255,"children":39257},{"className":39256,"style":2982},[2981],[],{"type":150,"tag":169,"props":39259,"children":39260},{"style":8452},[39261,39265],{"type":150,"tag":169,"props":39262,"children":39264},{"className":39263,"style":8427},[274],[],{"type":150,"tag":169,"props":39266,"children":39268},{"className":39267},[248],[39269,39274,39279],{"type":150,"tag":169,"props":39270,"children":39272},{"className":39271},[248],[39273],{"type":155,"value":39147},{"type":150,"tag":169,"props":39275,"children":39277},{"className":39276},[558],[39278],{"type":155,"value":8763},{"type":150,"tag":169,"props":39280,"children":39282},{"className":39281},[248,155],[39283],{"type":150,"tag":169,"props":39284,"children":39286},{"className":39285},[248],[39287],{"type":155,"value":23759},{"type":150,"tag":169,"props":39289,"children":39291},{"className":39290},[355],[39292],{"type":155,"value":358},{"type":150,"tag":169,"props":39294,"children":39296},{"className":39295},[259],[39297],{"type":150,"tag":169,"props":39298,"children":39300},{"className":39299,"style":23772},[264],[39301],{"type":150,"tag":169,"props":39302,"children":39303},{},[],{"type":150,"tag":169,"props":39305,"children":39307},{"className":39306},[1015,2660],[],{"type":150,"tag":169,"props":39309,"children":39311},{"className":39310,"style":839},[558],[],{"type":150,"tag":169,"props":39313,"children":39315},{"className":39314},[844],[39316],{"type":155,"value":1045},{"type":150,"tag":169,"props":39318,"children":39320},{"className":39319,"style":839},[558],[],{"type":150,"tag":169,"props":39322,"children":39324},{"className":39323},[237],[39325,39329,39334,39339],{"type":150,"tag":169,"props":39326,"children":39328},{"className":39327,"style":999},[242],[],{"type":150,"tag":169,"props":39330,"children":39332},{"className":39331},[248],[39333],{"type":155,"value":39171},{"type":150,"tag":169,"props":39335,"children":39337},{"className":39336},[558],[39338],{"type":155,"value":8763},{"type":150,"tag":169,"props":39340,"children":39342},{"className":39341},[248,155],[39343],{"type":150,"tag":169,"props":39344,"children":39346},{"className":39345},[248],[39347],{"type":155,"value":23821},{"type":150,"tag":151,"props":39349,"children":39350},{},[39351],{"type":155,"value":39352},"decode 侧每 token 要搬多少？16B activated，FP4 存储：",{"type":150,"tag":169,"props":39354,"children":39356},{"className":39355},[172],[39357],{"type":150,"tag":169,"props":39358,"children":39360},{"className":39359},[177],[39361,39412],{"type":150,"tag":169,"props":39362,"children":39364},{"className":39363},[182],[39365],{"type":150,"tag":185,"props":39366,"children":39367},{"xmlns":187,"display":188},[39368],{"type":150,"tag":191,"props":39369,"children":39370},{},[39371,39407],{"type":150,"tag":195,"props":39372,"children":39373},{},[39374,39378,39382,39386,39391,39395,39399,39403],{"type":150,"tag":411,"props":39375,"children":39376},{},[39377],{"type":155,"value":36853},{"type":150,"tag":217,"props":39379,"children":39380},{},[39381],{"type":155,"value":24918},{"type":150,"tag":417,"props":39383,"children":39384},{},[39385],{"type":155,"value":5554},{"type":150,"tag":411,"props":39387,"children":39388},{},[39389],{"type":155,"value":39390},"0.5",{"type":150,"tag":217,"props":39392,"children":39393},{},[39394],{"type":155,"value":24931},{"type":150,"tag":417,"props":39396,"children":39397},{},[39398],{"type":155,"value":1045},{"type":150,"tag":411,"props":39400,"children":39401},{},[39402],{"type":155,"value":9389},{"type":150,"tag":217,"props":39404,"children":39405},{},[39406],{"type":155,"value":29467},{"type":150,"tag":223,"props":39408,"children":39409},{"encoding":225},[39410],{"type":155,"value":39411},"16\\text{B} \\times 0.5\\ \\text{字节} = 8\\ \\text{GB}",{"type":150,"tag":169,"props":39413,"children":39415},{"className":39414,"ariaHidden":214},[232],[39416,39451,39491],{"type":150,"tag":169,"props":39417,"children":39419},{"className":39418},[237],[39420,39424,39429,39438,39442,39447],{"type":150,"tag":169,"props":39421,"children":39423},{"className":39422,"style":5644},[242],[],{"type":150,"tag":169,"props":39425,"children":39427},{"className":39426},[248],[39428],{"type":155,"value":36853},{"type":150,"tag":169,"props":39430,"children":39432},{"className":39431},[248,155],[39433],{"type":150,"tag":169,"props":39434,"children":39436},{"className":39435},[248],[39437],{"type":155,"value":24918},{"type":150,"tag":169,"props":39439,"children":39441},{"className":39440,"style":5167},[558],[],{"type":150,"tag":169,"props":39443,"children":39445},{"className":39444},[5172],[39446],{"type":155,"value":5554},{"type":150,"tag":169,"props":39448,"children":39450},{"className":39449,"style":5167},[558],[],{"type":150,"tag":169,"props":39452,"children":39454},{"className":39453},[237],[39455,39459,39464,39469,39478,39482,39487],{"type":150,"tag":169,"props":39456,"children":39458},{"className":39457,"style":5854},[242],[],{"type":150,"tag":169,"props":39460,"children":39462},{"className":39461},[248],[39463],{"type":155,"value":39390},{"type":150,"tag":169,"props":39465,"children":39467},{"className":39466},[558],[39468],{"type":155,"value":8763},{"type":150,"tag":169,"props":39470,"children":39472},{"className":39471},[248,155],[39473],{"type":150,"tag":169,"props":39474,"children":39476},{"className":39475},[248,297],[39477],{"type":155,"value":25038},{"type":150,"tag":169,"props":39479,"children":39481},{"className":39480,"style":839},[558],[],{"type":150,"tag":169,"props":39483,"children":39485},{"className":39484},[844],[39486],{"type":155,"value":1045},{"type":150,"tag":169,"props":39488,"children":39490},{"className":39489,"style":839},[558],[],{"type":150,"tag":169,"props":39492,"children":39494},{"className":39493},[237],[39495,39499,39504,39509],{"type":150,"tag":169,"props":39496,"children":39498},{"className":39497,"style":5854},[242],[],{"type":150,"tag":169,"props":39500,"children":39502},{"className":39501},[248],[39503],{"type":155,"value":9389},{"type":150,"tag":169,"props":39505,"children":39507},{"className":39506},[558],[39508],{"type":155,"value":8763},{"type":150,"tag":169,"props":39510,"children":39512},{"className":39511},[248,155],[39513],{"type":150,"tag":169,"props":39514,"children":39516},{"className":39515},[248],[39517],{"type":155,"value":29613},{"type":150,"tag":151,"props":39519,"children":39520},{},[39521,39523,39685],{"type":155,"value":39522},"KV 那边，1M context 时是 ",{"type":150,"tag":169,"props":39524,"children":39526},{"className":39525},[177],[39527,39575],{"type":150,"tag":169,"props":39528,"children":39530},{"className":39529},[182],[39531],{"type":150,"tag":185,"props":39532,"children":39533},{"xmlns":187},[39534],{"type":150,"tag":191,"props":39535,"children":39536},{},[39537,39570],{"type":150,"tag":195,"props":39538,"children":39539},{},[39540,39545,39549,39561,39565],{"type":150,"tag":411,"props":39541,"children":39542},{},[39543],{"type":155,"value":39544},"890",{"type":150,"tag":417,"props":39546,"children":39547},{},[39548],{"type":155,"value":5554},{"type":150,"tag":746,"props":39550,"children":39551},{},[39552,39556],{"type":150,"tag":411,"props":39553,"children":39554},{},[39555],{"type":155,"value":25792},{"type":150,"tag":411,"props":39557,"children":39558},{},[39559],{"type":155,"value":39560},"6",{"type":150,"tag":417,"props":39562,"children":39563},{},[39564],{"type":155,"value":6050},{"type":150,"tag":411,"props":39566,"children":39567},{},[39568],{"type":155,"value":39569},"0.93",{"type":150,"tag":223,"props":39571,"children":39572},{"encoding":225},[39573],{"type":155,"value":39574},"890 \\times 10^6 \\approx 0.93",{"type":150,"tag":169,"props":39576,"children":39578},{"className":39577,"ariaHidden":214},[232],[39579,39605,39672],{"type":150,"tag":169,"props":39580,"children":39582},{"className":39581},[237],[39583,39587,39592,39596,39601],{"type":150,"tag":169,"props":39584,"children":39586},{"className":39585,"style":5617},[242],[],{"type":150,"tag":169,"props":39588,"children":39590},{"className":39589},[248],[39591],{"type":155,"value":39544},{"type":150,"tag":169,"props":39593,"children":39595},{"className":39594,"style":5167},[558],[],{"type":150,"tag":169,"props":39597,"children":39599},{"className":39598},[5172],[39600],{"type":155,"value":5554},{"type":150,"tag":169,"props":39602,"children":39604},{"className":39603,"style":5167},[558],[],{"type":150,"tag":169,"props":39606,"children":39608},{"className":39607},[237],[39609,39613,39618,39659,39663,39668],{"type":150,"tag":169,"props":39610,"children":39612},{"className":39611,"style":7005},[242],[],{"type":150,"tag":169,"props":39614,"children":39616},{"className":39615},[248],[39617],{"type":155,"value":415},{"type":150,"tag":169,"props":39619,"children":39621},{"className":39620},[248],[39622,39627],{"type":150,"tag":169,"props":39623,"children":39625},{"className":39624},[248],[39626],{"type":155,"value":206},{"type":150,"tag":169,"props":39628,"children":39630},{"className":39629},[495],[39631],{"type":150,"tag":169,"props":39632,"children":39634},{"className":39633},[253],[39635],{"type":150,"tag":169,"props":39636,"children":39638},{"className":39637},[259],[39639],{"type":150,"tag":169,"props":39640,"children":39642},{"className":39641,"style":7005},[264],[39643],{"type":150,"tag":169,"props":39644,"children":39645},{"style":888},[39646,39650],{"type":150,"tag":169,"props":39647,"children":39649},{"className":39648,"style":517},[274],[],{"type":150,"tag":169,"props":39651,"children":39653},{"className":39652},[522,523,524,525],[39654],{"type":150,"tag":169,"props":39655,"children":39657},{"className":39656},[248,525],[39658],{"type":155,"value":39560},{"type":150,"tag":169,"props":39660,"children":39662},{"className":39661,"style":839},[558],[],{"type":150,"tag":169,"props":39664,"children":39666},{"className":39665},[844],[39667],{"type":155,"value":6050},{"type":150,"tag":169,"props":39669,"children":39671},{"className":39670,"style":839},[558],[],{"type":150,"tag":169,"props":39673,"children":39675},{"className":39674},[237],[39676,39680],{"type":150,"tag":169,"props":39677,"children":39679},{"className":39678,"style":5186},[242],[],{"type":150,"tag":169,"props":39681,"children":39683},{"className":39682},[248],[39684],{"type":155,"value":39569},{"type":155,"value":39686}," GB。加起来 8.93 GB。算术强度：",{"type":150,"tag":169,"props":39688,"children":39690},{"className":39689},[172],[39691],{"type":150,"tag":169,"props":39692,"children":39694},{"className":39693},[177],[39695,39768],{"type":150,"tag":169,"props":39696,"children":39698},{"className":39697},[182],[39699],{"type":150,"tag":185,"props":39700,"children":39701},{"xmlns":187,"display":188},[39702],{"type":150,"tag":191,"props":39703,"children":39704},{},[39705,39763],{"type":150,"tag":195,"props":39706,"children":39707},{},[39708,39712,39716,39750,39754,39759],{"type":150,"tag":217,"props":39709,"children":39710},{},[39711],{"type":155,"value":25141},{"type":150,"tag":417,"props":39713,"children":39714},{},[39715],{"type":155,"value":1045},{"type":150,"tag":1952,"props":39717,"children":39718},{},[39719,39738],{"type":150,"tag":195,"props":39720,"children":39721},{},[39722,39726,39730,39734],{"type":150,"tag":411,"props":39723,"children":39724},{},[39725],{"type":155,"value":433},{"type":150,"tag":417,"props":39727,"children":39728},{},[39729],{"type":155,"value":5554},{"type":150,"tag":411,"props":39731,"children":39732},{},[39733],{"type":155,"value":36853},{"type":150,"tag":217,"props":39735,"children":39736},{},[39737],{"type":155,"value":24918},{"type":150,"tag":195,"props":39739,"children":39740},{},[39741,39746],{"type":150,"tag":411,"props":39742,"children":39743},{},[39744],{"type":155,"value":39745},"8.93",{"type":150,"tag":217,"props":39747,"children":39748},{},[39749],{"type":155,"value":29467},{"type":150,"tag":417,"props":39751,"children":39752},{},[39753],{"type":155,"value":6050},{"type":150,"tag":411,"props":39755,"children":39756},{},[39757],{"type":155,"value":39758},"3.6",{"type":150,"tag":217,"props":39760,"children":39761},{},[39762],{"type":155,"value":23643},{"type":150,"tag":223,"props":39764,"children":39765},{"encoding":225},[39766],{"type":155,"value":39767},"\\text{AI} = \\frac{2 \\times 16\\text{B}}{8.93\\ \\text{GB}} \\approx 3.6\\ \\text{FLOP\u002Fbyte}",{"type":150,"tag":169,"props":39769,"children":39771},{"className":39770,"ariaHidden":214},[232],[39772,39802,39951],{"type":150,"tag":169,"props":39773,"children":39775},{"className":39774},[237],[39776,39780,39789,39793,39798],{"type":150,"tag":169,"props":39777,"children":39779},{"className":39778,"style":5854},[242],[],{"type":150,"tag":169,"props":39781,"children":39783},{"className":39782},[248,155],[39784],{"type":150,"tag":169,"props":39785,"children":39787},{"className":39786},[248],[39788],{"type":155,"value":25141},{"type":150,"tag":169,"props":39790,"children":39792},{"className":39791,"style":839},[558],[],{"type":150,"tag":169,"props":39794,"children":39796},{"className":39795},[844],[39797],{"type":155,"value":1045},{"type":150,"tag":169,"props":39799,"children":39801},{"className":39800,"style":839},[558],[],{"type":150,"tag":169,"props":39803,"children":39805},{"className":39804},[237],[39806,39810,39938,39942,39947],{"type":150,"tag":169,"props":39807,"children":39809},{"className":39808,"style":31864},[242],[],{"type":150,"tag":169,"props":39811,"children":39813},{"className":39812},[248],[39814,39818,39934],{"type":150,"tag":169,"props":39815,"children":39817},{"className":39816},[1004,2660],[],{"type":150,"tag":169,"props":39819,"children":39821},{"className":39820},[1952],[39822],{"type":150,"tag":169,"props":39823,"children":39825},{"className":39824},[253,254],[39826,39923],{"type":150,"tag":169,"props":39827,"children":39829},{"className":39828},[259],[39830,39918],{"type":150,"tag":169,"props":39831,"children":39833},{"className":39832,"style":23685},[264],[39834,39864,39875],{"type":150,"tag":169,"props":39835,"children":39836},{"style":8422},[39837,39841],{"type":150,"tag":169,"props":39838,"children":39840},{"className":39839,"style":8427},[274],[],{"type":150,"tag":169,"props":39842,"children":39844},{"className":39843},[248],[39845,39850,39855],{"type":150,"tag":169,"props":39846,"children":39848},{"className":39847},[248],[39849],{"type":155,"value":39745},{"type":150,"tag":169,"props":39851,"children":39853},{"className":39852},[558],[39854],{"type":155,"value":8763},{"type":150,"tag":169,"props":39856,"children":39858},{"className":39857},[248,155],[39859],{"type":150,"tag":169,"props":39860,"children":39862},{"className":39861},[248],[39863],{"type":155,"value":29613},{"type":150,"tag":169,"props":39865,"children":39866},{"style":8440},[39867,39871],{"type":150,"tag":169,"props":39868,"children":39870},{"className":39869,"style":8427},[274],[],{"type":150,"tag":169,"props":39872,"children":39874},{"className":39873,"style":2982},[2981],[],{"type":150,"tag":169,"props":39876,"children":39877},{"style":8452},[39878,39882],{"type":150,"tag":169,"props":39879,"children":39881},{"className":39880,"style":8427},[274],[],{"type":150,"tag":169,"props":39883,"children":39885},{"className":39884},[248],[39886,39891,39895,39900,39904,39909],{"type":150,"tag":169,"props":39887,"children":39889},{"className":39888},[248],[39890],{"type":155,"value":433},{"type":150,"tag":169,"props":39892,"children":39894},{"className":39893,"style":5167},[558],[],{"type":150,"tag":169,"props":39896,"children":39898},{"className":39897},[5172],[39899],{"type":155,"value":5554},{"type":150,"tag":169,"props":39901,"children":39903},{"className":39902,"style":5167},[558],[],{"type":150,"tag":169,"props":39905,"children":39907},{"className":39906},[248],[39908],{"type":155,"value":36853},{"type":150,"tag":169,"props":39910,"children":39912},{"className":39911},[248,155],[39913],{"type":150,"tag":169,"props":39914,"children":39916},{"className":39915},[248],[39917],{"type":155,"value":24918},{"type":150,"tag":169,"props":39919,"children":39921},{"className":39920},[355],[39922],{"type":155,"value":358},{"type":150,"tag":169,"props":39924,"children":39926},{"className":39925},[259],[39927],{"type":150,"tag":169,"props":39928,"children":39930},{"className":39929,"style":8484},[264],[39931],{"type":150,"tag":169,"props":39932,"children":39933},{},[],{"type":150,"tag":169,"props":39935,"children":39937},{"className":39936},[1015,2660],[],{"type":150,"tag":169,"props":39939,"children":39941},{"className":39940,"style":839},[558],[],{"type":150,"tag":169,"props":39943,"children":39945},{"className":39944},[844],[39946],{"type":155,"value":6050},{"type":150,"tag":169,"props":39948,"children":39950},{"className":39949,"style":839},[558],[],{"type":150,"tag":169,"props":39952,"children":39954},{"className":39953},[237],[39955,39959,39964,39969],{"type":150,"tag":169,"props":39956,"children":39958},{"className":39957,"style":999},[242],[],{"type":150,"tag":169,"props":39960,"children":39962},{"className":39961},[248],[39963],{"type":155,"value":39758},{"type":150,"tag":169,"props":39965,"children":39967},{"className":39966},[558],[39968],{"type":155,"value":8763},{"type":150,"tag":169,"props":39970,"children":39972},{"className":39971},[248,155],[39973],{"type":150,"tag":169,"props":39974,"children":39976},{"className":39975},[248],[39977],{"type":155,"value":23821},{"type":150,"tag":151,"props":39979,"children":39980},{},[39981],{"type":155,"value":39982},"按 8 TB\u002Fs 的带宽上限，单流大约 900 token\u002Fs。",{"type":150,"tag":151,"props":39984,"children":39985},{},[39986,39991],{"type":150,"tag":158,"props":39987,"children":39988},{},[39989],{"type":155,"value":39990},"3.6 对 1125，差了 300 倍。",{"type":155,"value":39992}," B200 的 FP4 算力有 99.7% 用不上。",{"type":150,"tag":151,"props":39994,"children":39995},{},[39996],{"type":155,"value":39997},"换权重精度结论也一样。KV 那 890 字节 \u002F token 已经是 FP4 下的实测值，所以只动权重：",{"type":150,"tag":22368,"props":39999,"children":40000},{},[40001,40031],{"type":150,"tag":22372,"props":40002,"children":40003},{},[40004],{"type":150,"tag":22376,"props":40005,"children":40006},{},[40007,40012,40017,40022,40026],{"type":150,"tag":22380,"props":40008,"children":40009},{},[40010],{"type":155,"value":40011},"权重精度",{"type":150,"tag":22380,"props":40013,"children":40014},{},[40015],{"type":155,"value":40016},"权重字节 \u002F token",{"type":150,"tag":22380,"props":40018,"children":40019},{},[40020],{"type":155,"value":40021},"拐点",{"type":150,"tag":22380,"props":40023,"children":40024},{},[40025],{"type":155,"value":27055},{"type":150,"tag":22380,"props":40027,"children":40028},{},[40029],{"type":155,"value":40030},"差几倍",{"type":150,"tag":22394,"props":40032,"children":40033},{},[40034,40060,40088],{"type":150,"tag":22376,"props":40035,"children":40036},{},[40037,40041,40046,40050,40055],{"type":150,"tag":22401,"props":40038,"children":40039},{},[40040],{"type":155,"value":39065},{"type":150,"tag":22401,"props":40042,"children":40043},{},[40044],{"type":155,"value":40045},"8.0 GB",{"type":150,"tag":22401,"props":40047,"children":40048},{},[40049],{"type":155,"value":39171},{"type":150,"tag":22401,"props":40051,"children":40052},{},[40053],{"type":155,"value":40054},"3.58",{"type":150,"tag":22401,"props":40056,"children":40057},{},[40058],{"type":155,"value":40059},"314×",{"type":150,"tag":22376,"props":40061,"children":40062},{},[40063,40068,40073,40078,40083],{"type":150,"tag":22401,"props":40064,"children":40065},{},[40066],{"type":155,"value":40067},"FP8",{"type":150,"tag":22401,"props":40069,"children":40070},{},[40071],{"type":155,"value":40072},"16.0 GB",{"type":150,"tag":22401,"props":40074,"children":40075},{},[40076],{"type":155,"value":40077},"562",{"type":150,"tag":22401,"props":40079,"children":40080},{},[40081],{"type":155,"value":40082},"1.89",{"type":150,"tag":22401,"props":40084,"children":40085},{},[40086],{"type":155,"value":40087},"298×",{"type":150,"tag":22376,"props":40089,"children":40090},{},[40091,40096,40101,40106,40111],{"type":150,"tag":22401,"props":40092,"children":40093},{},[40094],{"type":155,"value":40095},"BF16",{"type":150,"tag":22401,"props":40097,"children":40098},{},[40099],{"type":155,"value":40100},"32.0 GB",{"type":150,"tag":22401,"props":40102,"children":40103},{},[40104],{"type":155,"value":40105},"281",{"type":150,"tag":22401,"props":40107,"children":40108},{},[40109],{"type":155,"value":40110},"0.97",{"type":150,"tag":22401,"props":40112,"children":40113},{},[40114],{"type":155,"value":40115},"289×",{"type":150,"tag":151,"props":40117,"children":40118},{},[40119,40121,40370,40372,40377],{"type":155,"value":40120},"精度降一半，拐点翻一倍，但算术强度也跟着掉一半，两边抵消。根源是 B200 上这三个精度的「算力 × 每参字节」都是同一个数：",{"type":150,"tag":169,"props":40122,"children":40124},{"className":40123},[177],[40125,40197],{"type":150,"tag":169,"props":40126,"children":40128},{"className":40127},[182],[40129],{"type":150,"tag":185,"props":40130,"children":40131},{"xmlns":187},[40132],{"type":150,"tag":191,"props":40133,"children":40134},{},[40135,40192],{"type":150,"tag":195,"props":40136,"children":40137},{},[40138,40142,40146,40150,40154,40159,40163,40167,40171,40176,40180,40184,40188],{"type":150,"tag":411,"props":40139,"children":40140},{},[40141],{"type":155,"value":39147},{"type":150,"tag":417,"props":40143,"children":40144},{},[40145],{"type":155,"value":5554},{"type":150,"tag":411,"props":40147,"children":40148},{},[40149],{"type":155,"value":39390},{"type":150,"tag":417,"props":40151,"children":40152},{},[40153],{"type":155,"value":1045},{"type":150,"tag":411,"props":40155,"children":40156},{},[40157],{"type":155,"value":40158},"4500",{"type":150,"tag":417,"props":40160,"children":40161},{},[40162],{"type":155,"value":5554},{"type":150,"tag":411,"props":40164,"children":40165},{},[40166],{"type":155,"value":415},{"type":150,"tag":417,"props":40168,"children":40169},{},[40170],{"type":155,"value":1045},{"type":150,"tag":411,"props":40172,"children":40173},{},[40174],{"type":155,"value":40175},"2250",{"type":150,"tag":417,"props":40177,"children":40178},{},[40179],{"type":155,"value":5554},{"type":150,"tag":411,"props":40181,"children":40182},{},[40183],{"type":155,"value":433},{"type":150,"tag":417,"props":40185,"children":40186},{},[40187],{"type":155,"value":1045},{"type":150,"tag":411,"props":40189,"children":40190},{},[40191],{"type":155,"value":40158},{"type":150,"tag":223,"props":40193,"children":40194},{"encoding":225},[40195],{"type":155,"value":40196},"9000 \\times 0.5 = 4500 \\times 1 = 2250 \\times 2 = 4500",{"type":150,"tag":169,"props":40198,"children":40200},{"className":40199,"ariaHidden":214},[232],[40201,40227,40253,40279,40305,40331,40357],{"type":150,"tag":169,"props":40202,"children":40204},{"className":40203},[237],[40205,40209,40214,40218,40223],{"type":150,"tag":169,"props":40206,"children":40208},{"className":40207,"style":5617},[242],[],{"type":150,"tag":169,"props":40210,"children":40212},{"className":40211},[248],[40213],{"type":155,"value":39147},{"type":150,"tag":169,"props":40215,"children":40217},{"className":40216,"style":5167},[558],[],{"type":150,"tag":169,"props":40219,"children":40221},{"className":40220},[5172],[40222],{"type":155,"value":5554},{"type":150,"tag":169,"props":40224,"children":40226},{"className":40225,"style":5167},[558],[],{"type":150,"tag":169,"props":40228,"children":40230},{"className":40229},[237],[40231,40235,40240,40244,40249],{"type":150,"tag":169,"props":40232,"children":40234},{"className":40233,"style":5186},[242],[],{"type":150,"tag":169,"props":40236,"children":40238},{"className":40237},[248],[40239],{"type":155,"value":39390},{"type":150,"tag":169,"props":40241,"children":40243},{"className":40242,"style":839},[558],[],{"type":150,"tag":169,"props":40245,"children":40247},{"className":40246},[844],[40248],{"type":155,"value":1045},{"type":150,"tag":169,"props":40250,"children":40252},{"className":40251,"style":839},[558],[],{"type":150,"tag":169,"props":40254,"children":40256},{"className":40255},[237],[40257,40261,40266,40270,40275],{"type":150,"tag":169,"props":40258,"children":40260},{"className":40259,"style":5617},[242],[],{"type":150,"tag":169,"props":40262,"children":40264},{"className":40263},[248],[40265],{"type":155,"value":40158},{"type":150,"tag":169,"props":40267,"children":40269},{"className":40268,"style":5167},[558],[],{"type":150,"tag":169,"props":40271,"children":40273},{"className":40272},[5172],[40274],{"type":155,"value":5554},{"type":150,"tag":169,"props":40276,"children":40278},{"className":40277,"style":5167},[558],[],{"type":150,"tag":169,"props":40280,"children":40282},{"className":40281},[237],[40283,40287,40292,40296,40301],{"type":150,"tag":169,"props":40284,"children":40286},{"className":40285,"style":5186},[242],[],{"type":150,"tag":169,"props":40288,"children":40290},{"className":40289},[248],[40291],{"type":155,"value":415},{"type":150,"tag":169,"props":40293,"children":40295},{"className":40294,"style":839},[558],[],{"type":150,"tag":169,"props":40297,"children":40299},{"className":40298},[844],[40300],{"type":155,"value":1045},{"type":150,"tag":169,"props":40302,"children":40304},{"className":40303,"style":839},[558],[],{"type":150,"tag":169,"props":40306,"children":40308},{"className":40307},[237],[40309,40313,40318,40322,40327],{"type":150,"tag":169,"props":40310,"children":40312},{"className":40311,"style":5617},[242],[],{"type":150,"tag":169,"props":40314,"children":40316},{"className":40315},[248],[40317],{"type":155,"value":40175},{"type":150,"tag":169,"props":40319,"children":40321},{"className":40320,"style":5167},[558],[],{"type":150,"tag":169,"props":40323,"children":40325},{"className":40324},[5172],[40326],{"type":155,"value":5554},{"type":150,"tag":169,"props":40328,"children":40330},{"className":40329,"style":5167},[558],[],{"type":150,"tag":169,"props":40332,"children":40334},{"className":40333},[237],[40335,40339,40344,40348,40353],{"type":150,"tag":169,"props":40336,"children":40338},{"className":40337,"style":5186},[242],[],{"type":150,"tag":169,"props":40340,"children":40342},{"className":40341},[248],[40343],{"type":155,"value":433},{"type":150,"tag":169,"props":40345,"children":40347},{"className":40346,"style":839},[558],[],{"type":150,"tag":169,"props":40349,"children":40351},{"className":40350},[844],[40352],{"type":155,"value":1045},{"type":150,"tag":169,"props":40354,"children":40356},{"className":40355,"style":839},[558],[],{"type":150,"tag":169,"props":40358,"children":40360},{"className":40359},[237],[40361,40365],{"type":150,"tag":169,"props":40362,"children":40364},{"className":40363,"style":5186},[242],[],{"type":150,"tag":169,"props":40366,"children":40368},{"className":40367},[248],[40369],{"type":155,"value":40158},{"type":155,"value":40371},"。所以",{"type":150,"tag":158,"props":40373,"children":40374},{},[40375],{"type":155,"value":40376},"差多少倍基本由架构决定，不由精度决定",{"type":155,"value":904},{"type":150,"tag":151,"props":40379,"children":40380},{},[40381],{"type":155,"value":40382},"真正有意思的是这个拆分：",{"type":150,"tag":22368,"props":40384,"children":40385},{},[40386,40406],{"type":150,"tag":22372,"props":40387,"children":40388},{},[40389],{"type":150,"tag":22376,"props":40390,"children":40391},{},[40392,40397,40401],{"type":150,"tag":22380,"props":40393,"children":40394},{},[40395],{"type":155,"value":40396},"1M context 下每 token 的流量",{"type":150,"tag":22380,"props":40398,"children":40399},{},[40400],{"type":155,"value":25038},{"type":150,"tag":22380,"props":40402,"children":40403},{},[40404],{"type":155,"value":40405},"占比",{"type":150,"tag":22394,"props":40407,"children":40408},{},[40409,40426],{"type":150,"tag":22376,"props":40410,"children":40411},{},[40412,40417,40421],{"type":150,"tag":22401,"props":40413,"children":40414},{},[40415],{"type":155,"value":40416},"权重",{"type":150,"tag":22401,"props":40418,"children":40419},{},[40420],{"type":155,"value":40045},{"type":150,"tag":22401,"props":40422,"children":40423},{},[40424],{"type":155,"value":40425},"90%",{"type":150,"tag":22376,"props":40427,"children":40428},{},[40429,40434,40439],{"type":150,"tag":22401,"props":40430,"children":40431},{},[40432],{"type":155,"value":40433},"KV",{"type":150,"tag":22401,"props":40435,"children":40436},{},[40437],{"type":155,"value":40438},"0.93 GB",{"type":150,"tag":22401,"props":40440,"children":40441},{},[40442],{"type":155,"value":40443},"10%",{"type":150,"tag":151,"props":40445,"children":40446},{},[40447],{"type":150,"tag":158,"props":40448,"children":40449},{},[40450],{"type":155,"value":40451},"KV 已经不是瓶颈了。",{"type":150,"tag":151,"props":40453,"children":40454},{},[40455],{"type":155,"value":40456},"ds4.1f 把 KV 压到 890 字节 \u002F token 之后，就算跑到 1M context，它也只占总流量的 10%。再压一半，总时间只少 6%；压到四分之一，也只少 9%。",{"type":150,"tag":151,"props":40458,"children":40459},{},[40460],{"type":155,"value":40461},"而权重占 90%。把它减半，就是接近一倍的提速。",{"type":150,"tag":151,"props":40463,"children":40464},{},[40465],{"type":155,"value":40466},"字节数直接决定速度。decode 是 memory bound，每个 token 的耗时就是总字节除以带宽：",{"type":150,"tag":22368,"props":40468,"children":40469},{},[40470,40544],{"type":150,"tag":22372,"props":40471,"children":40472},{},[40473],{"type":150,"tag":22376,"props":40474,"children":40475},{},[40476,40521,40525,40529,40534,40539],{"type":150,"tag":22380,"props":40477,"children":40478},{},[40479],{"type":150,"tag":169,"props":40480,"children":40482},{"className":40481},[177],[40483,40504],{"type":150,"tag":169,"props":40484,"children":40486},{"className":40485},[182],[40487],{"type":150,"tag":185,"props":40488,"children":40489},{"xmlns":187},[40490],{"type":150,"tag":191,"props":40491,"children":40492},{},[40493,40500],{"type":150,"tag":195,"props":40494,"children":40495},{},[40496],{"type":150,"tag":405,"props":40497,"children":40498},{},[40499],{"type":155,"value":5591},{"type":150,"tag":223,"props":40501,"children":40502},{"encoding":225},[40503],{"type":155,"value":5591},{"type":150,"tag":169,"props":40505,"children":40507},{"className":40506,"ariaHidden":214},[232],[40508],{"type":150,"tag":169,"props":40509,"children":40511},{"className":40510},[237],[40512,40516],{"type":150,"tag":169,"props":40513,"children":40515},{"className":40514,"style":5854},[242],[],{"type":150,"tag":169,"props":40517,"children":40519},{"className":40518,"style":5780},[248,489],[40520],{"type":155,"value":5591},{"type":150,"tag":22380,"props":40522,"children":40523},{},[40524],{"type":155,"value":40416},{"type":150,"tag":22380,"props":40526,"children":40527},{},[40528],{"type":155,"value":40433},{"type":150,"tag":22380,"props":40530,"children":40531},{},[40532],{"type":155,"value":40533},"合计",{"type":150,"tag":22380,"props":40535,"children":40536},{},[40537],{"type":155,"value":40538},"每 token",{"type":150,"tag":22380,"props":40540,"children":40541},{},[40542],{"type":155,"value":40543},"单流 tok\u002Fs",{"type":150,"tag":22394,"props":40545,"children":40546},{},[40547,40578,40608,40639],{"type":150,"tag":22376,"props":40548,"children":40549},{},[40550,40554,40559,40564,40568,40573],{"type":150,"tag":22401,"props":40551,"children":40552},{},[40553],{"type":155,"value":27081},{"type":150,"tag":22401,"props":40555,"children":40556},{},[40557],{"type":155,"value":40558},"8.00 GB",{"type":150,"tag":22401,"props":40560,"children":40561},{},[40562],{"type":155,"value":40563},"0.004 GB",{"type":150,"tag":22401,"props":40565,"children":40566},{},[40567],{"type":155,"value":40558},{"type":150,"tag":22401,"props":40569,"children":40570},{},[40571],{"type":155,"value":40572},"1.00 ms",{"type":150,"tag":22401,"props":40574,"children":40575},{},[40576],{"type":155,"value":40577},"1000",{"type":150,"tag":22376,"props":40579,"children":40580},{},[40581,40585,40589,40594,40599,40603],{"type":150,"tag":22401,"props":40582,"children":40583},{},[40584],{"type":155,"value":27109},{"type":150,"tag":22401,"props":40586,"children":40587},{},[40588],{"type":155,"value":40558},{"type":150,"tag":22401,"props":40590,"children":40591},{},[40592],{"type":155,"value":40593},"0.029 GB",{"type":150,"tag":22401,"props":40595,"children":40596},{},[40597],{"type":155,"value":40598},"8.03 GB",{"type":150,"tag":22401,"props":40600,"children":40601},{},[40602],{"type":155,"value":40572},{"type":150,"tag":22401,"props":40604,"children":40605},{},[40606],{"type":155,"value":40607},"996",{"type":150,"tag":22376,"props":40609,"children":40610},{},[40611,40615,40619,40624,40629,40634],{"type":150,"tag":22401,"props":40612,"children":40613},{},[40614],{"type":155,"value":27136},{"type":150,"tag":22401,"props":40616,"children":40617},{},[40618],{"type":155,"value":40558},{"type":150,"tag":22401,"props":40620,"children":40621},{},[40622],{"type":155,"value":40623},"0.117 GB",{"type":150,"tag":22401,"props":40625,"children":40626},{},[40627],{"type":155,"value":40628},"8.12 GB",{"type":150,"tag":22401,"props":40630,"children":40631},{},[40632],{"type":155,"value":40633},"1.01 ms",{"type":150,"tag":22401,"props":40635,"children":40636},{},[40637],{"type":155,"value":40638},"986",{"type":150,"tag":22376,"props":40640,"children":40641},{},[40642,40646,40650,40655,40660,40665],{"type":150,"tag":22401,"props":40643,"children":40644},{},[40645],{"type":155,"value":27163},{"type":150,"tag":22401,"props":40647,"children":40648},{},[40649],{"type":155,"value":40558},{"type":150,"tag":22401,"props":40651,"children":40652},{},[40653],{"type":155,"value":40654},"0.933 GB",{"type":150,"tag":22401,"props":40656,"children":40657},{},[40658],{"type":155,"value":40659},"8.93 GB",{"type":150,"tag":22401,"props":40661,"children":40662},{},[40663],{"type":155,"value":40664},"1.12 ms",{"type":150,"tag":22401,"props":40666,"children":40667},{},[40668],{"type":155,"value":40669},"896",{"type":150,"tag":151,"props":40671,"children":40672},{},[40673,40678],{"type":150,"tag":158,"props":40674,"children":40675},{},[40676],{"type":155,"value":40677},"从 4k 到 1M，单流速度只掉 10%。",{"type":155,"value":40679}," 这才是 KV 压缩真正买到的东西。不是\"能跑长 context\"，是\"跑长 context 不掉速\"。",{"type":150,"tag":151,"props":40681,"children":40682},{},[40683],{"type":155,"value":40684},"再看容量。单卡 172 GB 可用显存，按 890 字节 \u002F token：",{"type":150,"tag":169,"props":40686,"children":40688},{"className":40687},[172],[40689],{"type":150,"tag":169,"props":40690,"children":40692},{"className":40691},[177],[40693,40752],{"type":150,"tag":169,"props":40694,"children":40696},{"className":40695},[182],[40697],{"type":150,"tag":185,"props":40698,"children":40699},{"xmlns":187,"display":188},[40700],{"type":150,"tag":191,"props":40701,"children":40702},{},[40703,40747],{"type":150,"tag":195,"props":40704,"children":40705},{},[40706,40733,40737,40742],{"type":150,"tag":1952,"props":40707,"children":40708},{},[40709,40721],{"type":150,"tag":195,"props":40710,"children":40711},{},[40712,40717],{"type":150,"tag":411,"props":40713,"children":40714},{},[40715],{"type":155,"value":40716},"172",{"type":150,"tag":217,"props":40718,"children":40719},{},[40720],{"type":155,"value":29467},{"type":150,"tag":195,"props":40722,"children":40723},{},[40724,40728],{"type":150,"tag":411,"props":40725,"children":40726},{},[40727],{"type":155,"value":39544},{"type":150,"tag":217,"props":40729,"children":40730},{},[40731],{"type":155,"value":40732}," B",{"type":150,"tag":417,"props":40734,"children":40735},{},[40736],{"type":155,"value":6050},{"type":150,"tag":411,"props":40738,"children":40739},{},[40740],{"type":155,"value":40741},"1.93",{"type":150,"tag":217,"props":40743,"children":40744},{},[40745],{"type":155,"value":40746}," 亿 token",{"type":150,"tag":223,"props":40748,"children":40749},{"encoding":225},[40750],{"type":155,"value":40751},"\\frac{172\\ \\text{GB}}{890\\ \\text{B}} \\approx 1.93\\ \\text{亿 token}",{"type":150,"tag":169,"props":40753,"children":40755},{"className":40754,"ariaHidden":214},[232],[40756,40892],{"type":150,"tag":169,"props":40757,"children":40759},{"className":40758},[237],[40760,40764,40879,40883,40888],{"type":150,"tag":169,"props":40761,"children":40763},{"className":40762,"style":31864},[242],[],{"type":150,"tag":169,"props":40765,"children":40767},{"className":40766},[248],[40768,40772,40875],{"type":150,"tag":169,"props":40769,"children":40771},{"className":40770},[1004,2660],[],{"type":150,"tag":169,"props":40773,"children":40775},{"className":40774},[1952],[40776],{"type":150,"tag":169,"props":40777,"children":40779},{"className":40778},[253,254],[40780,40864],{"type":150,"tag":169,"props":40781,"children":40783},{"className":40782},[259],[40784,40859],{"type":150,"tag":169,"props":40785,"children":40787},{"className":40786,"style":23685},[264],[40788,40818,40829],{"type":150,"tag":169,"props":40789,"children":40790},{"style":8422},[40791,40795],{"type":150,"tag":169,"props":40792,"children":40794},{"className":40793,"style":8427},[274],[],{"type":150,"tag":169,"props":40796,"children":40798},{"className":40797},[248],[40799,40804,40809],{"type":150,"tag":169,"props":40800,"children":40802},{"className":40801},[248],[40803],{"type":155,"value":39544},{"type":150,"tag":169,"props":40805,"children":40807},{"className":40806},[558],[40808],{"type":155,"value":8763},{"type":150,"tag":169,"props":40810,"children":40812},{"className":40811},[248,155],[40813],{"type":150,"tag":169,"props":40814,"children":40816},{"className":40815},[248],[40817],{"type":155,"value":24918},{"type":150,"tag":169,"props":40819,"children":40820},{"style":8440},[40821,40825],{"type":150,"tag":169,"props":40822,"children":40824},{"className":40823,"style":8427},[274],[],{"type":150,"tag":169,"props":40826,"children":40828},{"className":40827,"style":2982},[2981],[],{"type":150,"tag":169,"props":40830,"children":40831},{"style":8452},[40832,40836],{"type":150,"tag":169,"props":40833,"children":40835},{"className":40834,"style":8427},[274],[],{"type":150,"tag":169,"props":40837,"children":40839},{"className":40838},[248],[40840,40845,40850],{"type":150,"tag":169,"props":40841,"children":40843},{"className":40842},[248],[40844],{"type":155,"value":40716},{"type":150,"tag":169,"props":40846,"children":40848},{"className":40847},[558],[40849],{"type":155,"value":8763},{"type":150,"tag":169,"props":40851,"children":40853},{"className":40852},[248,155],[40854],{"type":150,"tag":169,"props":40855,"children":40857},{"className":40856},[248],[40858],{"type":155,"value":29613},{"type":150,"tag":169,"props":40860,"children":40862},{"className":40861},[355],[40863],{"type":155,"value":358},{"type":150,"tag":169,"props":40865,"children":40867},{"className":40866},[259],[40868],{"type":150,"tag":169,"props":40869,"children":40871},{"className":40870,"style":8484},[264],[40872],{"type":150,"tag":169,"props":40873,"children":40874},{},[],{"type":150,"tag":169,"props":40876,"children":40878},{"className":40877},[1015,2660],[],{"type":150,"tag":169,"props":40880,"children":40882},{"className":40881,"style":839},[558],[],{"type":150,"tag":169,"props":40884,"children":40886},{"className":40885},[844],[40887],{"type":155,"value":6050},{"type":150,"tag":169,"props":40889,"children":40891},{"className":40890,"style":839},[558],[],{"type":150,"tag":169,"props":40893,"children":40895},{"className":40894},[237],[40896,40900,40905,40910],{"type":150,"tag":169,"props":40897,"children":40899},{"className":40898,"style":942},[242],[],{"type":150,"tag":169,"props":40901,"children":40903},{"className":40902},[248],[40904],{"type":155,"value":40741},{"type":150,"tag":169,"props":40906,"children":40908},{"className":40907},[558],[40909],{"type":155,"value":8763},{"type":150,"tag":169,"props":40911,"children":40913},{"className":40912},[248,155],[40914,40920],{"type":150,"tag":169,"props":40915,"children":40917},{"className":40916},[248,297],[40918],{"type":155,"value":40919},"亿",{"type":150,"tag":169,"props":40921,"children":40923},{"className":40922},[248],[40924],{"type":155,"value":40925}," token",{"type":150,"tag":151,"props":40927,"children":40928},{},[40929],{"type":155,"value":40930},"一张卡装得下 1.93 亿 token 的 global KV。",{"type":150,"tag":151,"props":40932,"children":40933},{},[40934],{"type":155,"value":40935},"但容量不是硬上限。装不下就往下沉，host DRAM、SSD 一级一级放；请求再来时命中 prefix cache，把 KV 搬回 HBM 就能接着算。技术报告里那两个数本来就是按这个层级给的：常驻 HBM 的 global KV 是 890 字节 \u002F token，而落在 SSD 或 host memory 的 persistent KV 再压到 V4-Flash 的 1\u002F8。",{"type":150,"tag":151,"props":40937,"children":40938},{},[40939],{"type":150,"tag":158,"props":40940,"children":40941},{},[40942],{"type":155,"value":40943},"KV 这一层已经从\"装不下就崩\"的硬约束，变成了一个分层的存储问题。",{"type":150,"tag":151,"props":40945,"children":40946},{},[40947],{"type":155,"value":40948},"所以这条线走到这里，战场又换了一次位置：从\"压 KV\"换到\"压权重\"。前面几代解决的是历史存成什么样，接下来要解决的是 16B activated 这个数本身。",{"type":150,"tag":151,"props":40950,"children":40951},{},[40952],{"type":155,"value":40953},"batch 能不能救？批处理把权重摊薄，但同时把 KV 按 batch 复制：",{"type":150,"tag":169,"props":40955,"children":40957},{"className":40956},[172],[40958],{"type":150,"tag":169,"props":40959,"children":40961},{"className":40960},[177],[40962,41073],{"type":150,"tag":169,"props":40963,"children":40965},{"className":40964},[182],[40966],{"type":150,"tag":185,"props":40967,"children":40968},{"xmlns":187,"display":188},[40969],{"type":150,"tag":191,"props":40970,"children":40971},{},[40972,41068],{"type":150,"tag":195,"props":40973,"children":40974},{},[40975,40979,40983,40987,40991,40995,40999,41003],{"type":150,"tag":217,"props":40976,"children":40977},{},[40978],{"type":155,"value":25141},{"type":150,"tag":417,"props":40980,"children":40981},{"stretchy":207},[40982],{"type":155,"value":972},{"type":150,"tag":405,"props":40984,"children":40985},{},[40986],{"type":155,"value":24918},{"type":150,"tag":417,"props":40988,"children":40989},{"separator":214},[40990],{"type":155,"value":421},{"type":150,"tag":405,"props":40992,"children":40993},{},[40994],{"type":155,"value":5591},{"type":150,"tag":417,"props":40996,"children":40997},{"stretchy":207},[40998],{"type":155,"value":982},{"type":150,"tag":417,"props":41000,"children":41001},{},[41002],{"type":155,"value":1045},{"type":150,"tag":1952,"props":41004,"children":41005},{},[41006,41033],{"type":150,"tag":195,"props":41007,"children":41008},{},[41009,41013,41017,41021,41025,41029],{"type":150,"tag":405,"props":41010,"children":41011},{},[41012],{"type":155,"value":24918},{"type":150,"tag":417,"props":41014,"children":41015},{},[41016],{"type":155,"value":5554},{"type":150,"tag":411,"props":41018,"children":41019},{},[41020],{"type":155,"value":433},{"type":150,"tag":417,"props":41022,"children":41023},{},[41024],{"type":155,"value":5554},{"type":150,"tag":411,"props":41026,"children":41027},{},[41028],{"type":155,"value":36853},{"type":150,"tag":217,"props":41030,"children":41031},{},[41032],{"type":155,"value":24918},{"type":150,"tag":195,"props":41034,"children":41035},{},[41036,41040,41044,41048,41052,41056,41060,41064],{"type":150,"tag":411,"props":41037,"children":41038},{},[41039],{"type":155,"value":9389},{"type":150,"tag":217,"props":41041,"children":41042},{},[41043],{"type":155,"value":29467},{"type":150,"tag":417,"props":41045,"children":41046},{},[41047],{"type":155,"value":5136},{"type":150,"tag":405,"props":41049,"children":41050},{},[41051],{"type":155,"value":24918},{"type":150,"tag":417,"props":41053,"children":41054},{},[41055],{"type":155,"value":5554},{"type":150,"tag":411,"props":41057,"children":41058},{},[41059],{"type":155,"value":39544},{"type":150,"tag":417,"props":41061,"children":41062},{},[41063],{"type":155,"value":5554},{"type":150,"tag":405,"props":41065,"children":41066},{},[41067],{"type":155,"value":5591},{"type":150,"tag":223,"props":41069,"children":41070},{"encoding":225},[41071],{"type":155,"value":41072},"\\text{AI}(B, N) = \\frac{B \\times 2 \\times 16\\text{B}}{8\\ \\text{GB} + B \\times 890 \\times N}",{"type":150,"tag":169,"props":41074,"children":41076},{"className":41075,"ariaHidden":214},[232],[41077,41136],{"type":150,"tag":169,"props":41078,"children":41080},{"className":41079},[237],[41081,41085,41094,41099,41104,41109,41113,41118,41123,41127,41132],{"type":150,"tag":169,"props":41082,"children":41084},{"className":41083,"style":999},[242],[],{"type":150,"tag":169,"props":41086,"children":41088},{"className":41087},[248,155],[41089],{"type":150,"tag":169,"props":41090,"children":41092},{"className":41091},[248],[41093],{"type":155,"value":25141},{"type":150,"tag":169,"props":41095,"children":41097},{"className":41096},[1004],[41098],{"type":155,"value":972},{"type":150,"tag":169,"props":41100,"children":41102},{"className":41101,"style":27238},[248,489],[41103],{"type":155,"value":24918},{"type":150,"tag":169,"props":41105,"children":41107},{"className":41106},[552],[41108],{"type":155,"value":421},{"type":150,"tag":169,"props":41110,"children":41112},{"className":41111,"style":559},[558],[],{"type":150,"tag":169,"props":41114,"children":41116},{"className":41115,"style":5780},[248,489],[41117],{"type":155,"value":5591},{"type":150,"tag":169,"props":41119,"children":41121},{"className":41120},[1015],[41122],{"type":155,"value":982},{"type":150,"tag":169,"props":41124,"children":41126},{"className":41125,"style":839},[558],[],{"type":150,"tag":169,"props":41128,"children":41130},{"className":41129},[844],[41131],{"type":155,"value":1045},{"type":150,"tag":169,"props":41133,"children":41135},{"className":41134,"style":839},[558],[],{"type":150,"tag":169,"props":41137,"children":41139},{"className":41138},[237],[41140,41145],{"type":150,"tag":169,"props":41141,"children":41144},{"className":41142,"style":41143},[242],"height:2.1297em;vertical-align:-0.7693em;",[],{"type":150,"tag":169,"props":41146,"children":41148},{"className":41147},[248],[41149,41153,41341],{"type":150,"tag":169,"props":41150,"children":41152},{"className":41151},[1004,2660],[],{"type":150,"tag":169,"props":41154,"children":41156},{"className":41155},[1952],[41157],{"type":150,"tag":169,"props":41158,"children":41160},{"className":41159},[253,254],[41161,41330],{"type":150,"tag":169,"props":41162,"children":41164},{"className":41163},[259],[41165,41325],{"type":150,"tag":169,"props":41166,"children":41168},{"className":41167,"style":23685},[264],[41169,41253,41264],{"type":150,"tag":169,"props":41170,"children":41171},{"style":8422},[41172,41176],{"type":150,"tag":169,"props":41173,"children":41175},{"className":41174,"style":8427},[274],[],{"type":150,"tag":169,"props":41177,"children":41179},{"className":41178},[248],[41180,41185,41190,41199,41203,41208,41212,41217,41221,41226,41230,41235,41239,41244,41248],{"type":150,"tag":169,"props":41181,"children":41183},{"className":41182},[248],[41184],{"type":155,"value":9389},{"type":150,"tag":169,"props":41186,"children":41188},{"className":41187},[558],[41189],{"type":155,"value":8763},{"type":150,"tag":169,"props":41191,"children":41193},{"className":41192},[248,155],[41194],{"type":150,"tag":169,"props":41195,"children":41197},{"className":41196},[248],[41198],{"type":155,"value":29613},{"type":150,"tag":169,"props":41200,"children":41202},{"className":41201,"style":5167},[558],[],{"type":150,"tag":169,"props":41204,"children":41206},{"className":41205},[5172],[41207],{"type":155,"value":5136},{"type":150,"tag":169,"props":41209,"children":41211},{"className":41210,"style":5167},[558],[],{"type":150,"tag":169,"props":41213,"children":41215},{"className":41214,"style":27238},[248,489],[41216],{"type":155,"value":24918},{"type":150,"tag":169,"props":41218,"children":41220},{"className":41219,"style":5167},[558],[],{"type":150,"tag":169,"props":41222,"children":41224},{"className":41223},[5172],[41225],{"type":155,"value":5554},{"type":150,"tag":169,"props":41227,"children":41229},{"className":41228,"style":5167},[558],[],{"type":150,"tag":169,"props":41231,"children":41233},{"className":41232},[248],[41234],{"type":155,"value":39544},{"type":150,"tag":169,"props":41236,"children":41238},{"className":41237,"style":5167},[558],[],{"type":150,"tag":169,"props":41240,"children":41242},{"className":41241},[5172],[41243],{"type":155,"value":5554},{"type":150,"tag":169,"props":41245,"children":41247},{"className":41246,"style":5167},[558],[],{"type":150,"tag":169,"props":41249,"children":41251},{"className":41250,"style":5780},[248,489],[41252],{"type":155,"value":5591},{"type":150,"tag":169,"props":41254,"children":41255},{"style":8440},[41256,41260],{"type":150,"tag":169,"props":41257,"children":41259},{"className":41258,"style":8427},[274],[],{"type":150,"tag":169,"props":41261,"children":41263},{"className":41262,"style":2982},[2981],[],{"type":150,"tag":169,"props":41265,"children":41266},{"style":8452},[41267,41271],{"type":150,"tag":169,"props":41268,"children":41270},{"className":41269,"style":8427},[274],[],{"type":150,"tag":169,"props":41272,"children":41274},{"className":41273},[248],[41275,41280,41284,41289,41293,41298,41302,41307,41311,41316],{"type":150,"tag":169,"props":41276,"children":41278},{"className":41277,"style":27238},[248,489],[41279],{"type":155,"value":24918},{"type":150,"tag":169,"props":41281,"children":41283},{"className":41282,"style":5167},[558],[],{"type":150,"tag":169,"props":41285,"children":41287},{"className":41286},[5172],[41288],{"type":155,"value":5554},{"type":150,"tag":169,"props":41290,"children":41292},{"className":41291,"style":5167},[558],[],{"type":150,"tag":169,"props":41294,"children":41296},{"className":41295},[248],[41297],{"type":155,"value":433},{"type":150,"tag":169,"props":41299,"children":41301},{"className":41300,"style":5167},[558],[],{"type":150,"tag":169,"props":41303,"children":41305},{"className":41304},[5172],[41306],{"type":155,"value":5554},{"type":150,"tag":169,"props":41308,"children":41310},{"className":41309,"style":5167},[558],[],{"type":150,"tag":169,"props":41312,"children":41314},{"className":41313},[248],[41315],{"type":155,"value":36853},{"type":150,"tag":169,"props":41317,"children":41319},{"className":41318},[248,155],[41320],{"type":150,"tag":169,"props":41321,"children":41323},{"className":41322},[248],[41324],{"type":155,"value":24918},{"type":150,"tag":169,"props":41326,"children":41328},{"className":41327},[355],[41329],{"type":155,"value":358},{"type":150,"tag":169,"props":41331,"children":41333},{"className":41332},[259],[41334],{"type":150,"tag":169,"props":41335,"children":41337},{"className":41336,"style":26239},[264],[41338],{"type":150,"tag":169,"props":41339,"children":41340},{},[],{"type":150,"tag":169,"props":41342,"children":41344},{"className":41343},[1015,2660],[],{"type":150,"tag":151,"props":41346,"children":41347},{},[41348],{"type":155,"value":41349},"代进不同长度：",{"type":150,"tag":22368,"props":41351,"children":41352},{},[41353,41799],{"type":150,"tag":22372,"props":41354,"children":41355},{},[41356],{"type":150,"tag":22376,"props":41357,"children":41358},{},[41359,41404,41483,41562,41641,41720],{"type":150,"tag":22380,"props":41360,"children":41361},{},[41362],{"type":150,"tag":169,"props":41363,"children":41365},{"className":41364},[177],[41366,41387],{"type":150,"tag":169,"props":41367,"children":41369},{"className":41368},[182],[41370],{"type":150,"tag":185,"props":41371,"children":41372},{"xmlns":187},[41373],{"type":150,"tag":191,"props":41374,"children":41375},{},[41376,41383],{"type":150,"tag":195,"props":41377,"children":41378},{},[41379],{"type":150,"tag":405,"props":41380,"children":41381},{},[41382],{"type":155,"value":5591},{"type":150,"tag":223,"props":41384,"children":41385},{"encoding":225},[41386],{"type":155,"value":5591},{"type":150,"tag":169,"props":41388,"children":41390},{"className":41389,"ariaHidden":214},[232],[41391],{"type":150,"tag":169,"props":41392,"children":41394},{"className":41393},[237],[41395,41399],{"type":150,"tag":169,"props":41396,"children":41398},{"className":41397,"style":5854},[242],[],{"type":150,"tag":169,"props":41400,"children":41402},{"className":41401,"style":5780},[248,489],[41403],{"type":155,"value":5591},{"type":150,"tag":22380,"props":41405,"children":41406},{},[41407],{"type":150,"tag":169,"props":41408,"children":41410},{"className":41409},[177],[41411,41440],{"type":150,"tag":169,"props":41412,"children":41414},{"className":41413},[182],[41415],{"type":150,"tag":185,"props":41416,"children":41417},{"xmlns":187},[41418],{"type":150,"tag":191,"props":41419,"children":41420},{},[41421,41436],{"type":150,"tag":195,"props":41422,"children":41423},{},[41424,41428,41432],{"type":150,"tag":405,"props":41425,"children":41426},{},[41427],{"type":155,"value":24918},{"type":150,"tag":417,"props":41429,"children":41430},{},[41431],{"type":155,"value":1045},{"type":150,"tag":411,"props":41433,"children":41434},{},[41435],{"type":155,"value":415},{"type":150,"tag":223,"props":41437,"children":41438},{"encoding":225},[41439],{"type":155,"value":28388},{"type":150,"tag":169,"props":41441,"children":41443},{"className":41442,"ariaHidden":214},[232],[41444,41470],{"type":150,"tag":169,"props":41445,"children":41447},{"className":41446},[237],[41448,41452,41457,41461,41466],{"type":150,"tag":169,"props":41449,"children":41451},{"className":41450,"style":5854},[242],[],{"type":150,"tag":169,"props":41453,"children":41455},{"className":41454,"style":27238},[248,489],[41456],{"type":155,"value":24918},{"type":150,"tag":169,"props":41458,"children":41460},{"className":41459,"style":839},[558],[],{"type":150,"tag":169,"props":41462,"children":41464},{"className":41463},[844],[41465],{"type":155,"value":1045},{"type":150,"tag":169,"props":41467,"children":41469},{"className":41468,"style":839},[558],[],{"type":150,"tag":169,"props":41471,"children":41473},{"className":41472},[237],[41474,41478],{"type":150,"tag":169,"props":41475,"children":41477},{"className":41476,"style":5186},[242],[],{"type":150,"tag":169,"props":41479,"children":41481},{"className":41480},[248],[41482],{"type":155,"value":415},{"type":150,"tag":22380,"props":41484,"children":41485},{},[41486],{"type":150,"tag":169,"props":41487,"children":41489},{"className":41488},[177],[41490,41519],{"type":150,"tag":169,"props":41491,"children":41493},{"className":41492},[182],[41494],{"type":150,"tag":185,"props":41495,"children":41496},{"xmlns":187},[41497],{"type":150,"tag":191,"props":41498,"children":41499},{},[41500,41515],{"type":150,"tag":195,"props":41501,"children":41502},{},[41503,41507,41511],{"type":150,"tag":405,"props":41504,"children":41505},{},[41506],{"type":155,"value":24918},{"type":150,"tag":417,"props":41508,"children":41509},{},[41510],{"type":155,"value":1045},{"type":150,"tag":411,"props":41512,"children":41513},{},[41514],{"type":155,"value":9389},{"type":150,"tag":223,"props":41516,"children":41517},{"encoding":225},[41518],{"type":155,"value":28468},{"type":150,"tag":169,"props":41520,"children":41522},{"className":41521,"ariaHidden":214},[232],[41523,41549],{"type":150,"tag":169,"props":41524,"children":41526},{"className":41525},[237],[41527,41531,41536,41540,41545],{"type":150,"tag":169,"props":41528,"children":41530},{"className":41529,"style":5854},[242],[],{"type":150,"tag":169,"props":41532,"children":41534},{"className":41533,"style":27238},[248,489],[41535],{"type":155,"value":24918},{"type":150,"tag":169,"props":41537,"children":41539},{"className":41538,"style":839},[558],[],{"type":150,"tag":169,"props":41541,"children":41543},{"className":41542},[844],[41544],{"type":155,"value":1045},{"type":150,"tag":169,"props":41546,"children":41548},{"className":41547,"style":839},[558],[],{"type":150,"tag":169,"props":41550,"children":41552},{"className":41551},[237],[41553,41557],{"type":150,"tag":169,"props":41554,"children":41556},{"className":41555,"style":5186},[242],[],{"type":150,"tag":169,"props":41558,"children":41560},{"className":41559},[248],[41561],{"type":155,"value":9389},{"type":150,"tag":22380,"props":41563,"children":41564},{},[41565],{"type":150,"tag":169,"props":41566,"children":41568},{"className":41567},[177],[41569,41598],{"type":150,"tag":169,"props":41570,"children":41572},{"className":41571},[182],[41573],{"type":150,"tag":185,"props":41574,"children":41575},{"xmlns":187},[41576],{"type":150,"tag":191,"props":41577,"children":41578},{},[41579,41594],{"type":150,"tag":195,"props":41580,"children":41581},{},[41582,41586,41590],{"type":150,"tag":405,"props":41583,"children":41584},{},[41585],{"type":155,"value":24918},{"type":150,"tag":417,"props":41587,"children":41588},{},[41589],{"type":155,"value":1045},{"type":150,"tag":411,"props":41591,"children":41592},{},[41593],{"type":155,"value":20836},{"type":150,"tag":223,"props":41595,"children":41596},{"encoding":225},[41597],{"type":155,"value":28548},{"type":150,"tag":169,"props":41599,"children":41601},{"className":41600,"ariaHidden":214},[232],[41602,41628],{"type":150,"tag":169,"props":41603,"children":41605},{"className":41604},[237],[41606,41610,41615,41619,41624],{"type":150,"tag":169,"props":41607,"children":41609},{"className":41608,"style":5854},[242],[],{"type":150,"tag":169,"props":41611,"children":41613},{"className":41612,"style":27238},[248,489],[41614],{"type":155,"value":24918},{"type":150,"tag":169,"props":41616,"children":41618},{"className":41617,"style":839},[558],[],{"type":150,"tag":169,"props":41620,"children":41622},{"className":41621},[844],[41623],{"type":155,"value":1045},{"type":150,"tag":169,"props":41625,"children":41627},{"className":41626,"style":839},[558],[],{"type":150,"tag":169,"props":41629,"children":41631},{"className":41630},[237],[41632,41636],{"type":150,"tag":169,"props":41633,"children":41635},{"className":41634,"style":5186},[242],[],{"type":150,"tag":169,"props":41637,"children":41639},{"className":41638},[248],[41640],{"type":155,"value":20836},{"type":150,"tag":22380,"props":41642,"children":41643},{},[41644],{"type":150,"tag":169,"props":41645,"children":41647},{"className":41646},[177],[41648,41677],{"type":150,"tag":169,"props":41649,"children":41651},{"className":41650},[182],[41652],{"type":150,"tag":185,"props":41653,"children":41654},{"xmlns":187},[41655],{"type":150,"tag":191,"props":41656,"children":41657},{},[41658,41673],{"type":150,"tag":195,"props":41659,"children":41660},{},[41661,41665,41669],{"type":150,"tag":405,"props":41662,"children":41663},{},[41664],{"type":155,"value":24918},{"type":150,"tag":417,"props":41666,"children":41667},{},[41668],{"type":155,"value":1045},{"type":150,"tag":411,"props":41670,"children":41671},{},[41672],{"type":155,"value":20981},{"type":150,"tag":223,"props":41674,"children":41675},{"encoding":225},[41676],{"type":155,"value":28628},{"type":150,"tag":169,"props":41678,"children":41680},{"className":41679,"ariaHidden":214},[232],[41681,41707],{"type":150,"tag":169,"props":41682,"children":41684},{"className":41683},[237],[41685,41689,41694,41698,41703],{"type":150,"tag":169,"props":41686,"children":41688},{"className":41687,"style":5854},[242],[],{"type":150,"tag":169,"props":41690,"children":41692},{"className":41691,"style":27238},[248,489],[41693],{"type":155,"value":24918},{"type":150,"tag":169,"props":41695,"children":41697},{"className":41696,"style":839},[558],[],{"type":150,"tag":169,"props":41699,"children":41701},{"className":41700},[844],[41702],{"type":155,"value":1045},{"type":150,"tag":169,"props":41704,"children":41706},{"className":41705,"style":839},[558],[],{"type":150,"tag":169,"props":41708,"children":41710},{"className":41709},[237],[41711,41715],{"type":150,"tag":169,"props":41712,"children":41714},{"className":41713,"style":5186},[242],[],{"type":150,"tag":169,"props":41716,"children":41718},{"className":41717},[248],[41719],{"type":155,"value":20981},{"type":150,"tag":22380,"props":41721,"children":41722},{},[41723],{"type":150,"tag":169,"props":41724,"children":41726},{"className":41725},[177],[41727,41756],{"type":150,"tag":169,"props":41728,"children":41730},{"className":41729},[182],[41731],{"type":150,"tag":185,"props":41732,"children":41733},{"xmlns":187},[41734],{"type":150,"tag":191,"props":41735,"children":41736},{},[41737,41752],{"type":150,"tag":195,"props":41738,"children":41739},{},[41740,41744,41748],{"type":150,"tag":405,"props":41741,"children":41742},{},[41743],{"type":155,"value":24918},{"type":150,"tag":417,"props":41745,"children":41746},{},[41747],{"type":155,"value":12954},{"type":150,"tag":405,"props":41749,"children":41750},{"mathvariant":1796},[41751],{"type":155,"value":26346},{"type":150,"tag":223,"props":41753,"children":41754},{"encoding":225},[41755],{"type":155,"value":28708},{"type":150,"tag":169,"props":41757,"children":41759},{"className":41758,"ariaHidden":214},[232],[41760,41786],{"type":150,"tag":169,"props":41761,"children":41763},{"className":41762},[237],[41764,41768,41773,41777,41782],{"type":150,"tag":169,"props":41765,"children":41767},{"className":41766,"style":5854},[242],[],{"type":150,"tag":169,"props":41769,"children":41771},{"className":41770,"style":27238},[248,489],[41772],{"type":155,"value":24918},{"type":150,"tag":169,"props":41774,"children":41776},{"className":41775,"style":839},[558],[],{"type":150,"tag":169,"props":41778,"children":41780},{"className":41779},[844],[41781],{"type":155,"value":12954},{"type":150,"tag":169,"props":41783,"children":41785},{"className":41784,"style":839},[558],[],{"type":150,"tag":169,"props":41787,"children":41789},{"className":41788},[237],[41790,41794],{"type":150,"tag":169,"props":41791,"children":41793},{"className":41792,"style":1094},[242],[],{"type":150,"tag":169,"props":41795,"children":41797},{"className":41796},[248],[41798],{"type":155,"value":26346},{"type":150,"tag":22394,"props":41800,"children":41801},{},[41802,41839,41870,41901],{"type":150,"tag":22376,"props":41803,"children":41804},{},[41805,41809,41814,41818,41823,41831],{"type":150,"tag":22401,"props":41806,"children":41807},{},[41808],{"type":155,"value":27081},{"type":150,"tag":22401,"props":41810,"children":41811},{},[41812],{"type":155,"value":41813},"4.0",{"type":150,"tag":22401,"props":41815,"children":41816},{},[41817],{"type":155,"value":36879},{"type":150,"tag":22401,"props":41819,"children":41820},{},[41821],{"type":155,"value":41822},"249",{"type":150,"tag":22401,"props":41824,"children":41825},{},[41826],{"type":150,"tag":158,"props":41827,"children":41828},{},[41829],{"type":155,"value":41830},"1661",{"type":150,"tag":22401,"props":41832,"children":41833},{},[41834],{"type":150,"tag":158,"props":41835,"children":41836},{},[41837],{"type":155,"value":41838},"8778",{"type":150,"tag":22376,"props":41840,"children":41841},{},[41842,41846,41850,41855,41860,41865],{"type":150,"tag":22401,"props":41843,"children":41844},{},[41845],{"type":155,"value":27109},{"type":150,"tag":22401,"props":41847,"children":41848},{},[41849],{"type":155,"value":41813},{"type":150,"tag":22401,"props":41851,"children":41852},{},[41853],{"type":155,"value":41854},"31",{"type":150,"tag":22401,"props":41856,"children":41857},{},[41858],{"type":155,"value":41859},"208",{"type":150,"tag":22401,"props":41861,"children":41862},{},[41863],{"type":155,"value":41864},"715",{"type":150,"tag":22401,"props":41866,"children":41867},{},[41868],{"type":155,"value":41869},"1097",{"type":150,"tag":22376,"props":41871,"children":41872},{},[41873,41877,41882,41887,41892,41896],{"type":150,"tag":22401,"props":41874,"children":41875},{},[41876],{"type":155,"value":27136},{"type":150,"tag":22401,"props":41878,"children":41879},{},[41880],{"type":155,"value":41881},"3.9",{"type":150,"tag":22401,"props":41883,"children":41884},{},[41885],{"type":155,"value":41886},"29",{"type":150,"tag":22401,"props":41888,"children":41889},{},[41890],{"type":155,"value":41891},"132",{"type":150,"tag":22401,"props":41893,"children":41894},{},[41895],{"type":155,"value":26415},{"type":150,"tag":22401,"props":41897,"children":41898},{},[41899],{"type":155,"value":41900},"274",{"type":150,"tag":22376,"props":41902,"children":41903},{},[41904,41908,41912,41917,41922,41927],{"type":150,"tag":22401,"props":41905,"children":41906},{},[41907],{"type":155,"value":27163},{"type":150,"tag":22401,"props":41909,"children":41910},{},[41911],{"type":155,"value":39758},{"type":150,"tag":22401,"props":41913,"children":41914},{},[41915],{"type":155,"value":41916},"17",{"type":150,"tag":22401,"props":41918,"children":41919},{},[41920],{"type":155,"value":41921},"30",{"type":150,"tag":22401,"props":41923,"children":41924},{},[41925],{"type":155,"value":41926},"34",{"type":150,"tag":22401,"props":41928,"children":41929},{},[41930],{"type":155,"value":41926},{"type":150,"tag":151,"props":41932,"children":41933},{},[41934],{"type":155,"value":41935},"（加粗的是越过 1125 的格子。）",{"type":150,"tag":151,"props":41937,"children":41938},{},[41939,41941,42020,42021,42098,42100,42190,42192,42268],{"type":155,"value":41940},"整张表只有两格越线，都在 4k 那一行，而且要 ",{"type":150,"tag":169,"props":41942,"children":41944},{"className":41943},[177],[41945,41976],{"type":150,"tag":169,"props":41946,"children":41948},{"className":41947},[182],[41949],{"type":150,"tag":185,"props":41950,"children":41951},{"xmlns":187},[41952],{"type":150,"tag":191,"props":41953,"children":41954},{},[41955,41971],{"type":150,"tag":195,"props":41956,"children":41957},{},[41958,41962,41967],{"type":150,"tag":405,"props":41959,"children":41960},{},[41961],{"type":155,"value":24918},{"type":150,"tag":417,"props":41963,"children":41964},{},[41965],{"type":155,"value":41966},"≥",{"type":150,"tag":411,"props":41968,"children":41969},{},[41970],{"type":155,"value":20981},{"type":150,"tag":223,"props":41972,"children":41973},{"encoding":225},[41974],{"type":155,"value":41975},"B \\ge 512",{"type":150,"tag":169,"props":41977,"children":41979},{"className":41978,"ariaHidden":214},[232],[41980,42007],{"type":150,"tag":169,"props":41981,"children":41983},{"className":41982},[237],[41984,41989,41994,41998,42003],{"type":150,"tag":169,"props":41985,"children":41988},{"className":41986,"style":41987},[242],"height:0.8193em;vertical-align:-0.136em;",[],{"type":150,"tag":169,"props":41990,"children":41992},{"className":41991,"style":27238},[248,489],[41993],{"type":155,"value":24918},{"type":150,"tag":169,"props":41995,"children":41997},{"className":41996,"style":839},[558],[],{"type":150,"tag":169,"props":41999,"children":42001},{"className":42000},[844],[42002],{"type":155,"value":41966},{"type":150,"tag":169,"props":42004,"children":42006},{"className":42005,"style":839},[558],[],{"type":150,"tag":169,"props":42008,"children":42010},{"className":42009},[237],[42011,42015],{"type":150,"tag":169,"props":42012,"children":42014},{"className":42013,"style":5186},[242],[],{"type":150,"tag":169,"props":42016,"children":42018},{"className":42017},[248],[42019],{"type":155,"value":20981},{"type":155,"value":904},{"type":150,"tag":169,"props":42022,"children":42024},{"className":42023},[177],[42025,42055],{"type":150,"tag":169,"props":42026,"children":42028},{"className":42027},[182],[42029],{"type":150,"tag":185,"props":42030,"children":42031},{"xmlns":187},[42032],{"type":150,"tag":191,"props":42033,"children":42034},{},[42035,42050],{"type":150,"tag":195,"props":42036,"children":42037},{},[42038,42042,42046],{"type":150,"tag":405,"props":42039,"children":42040},{},[42041],{"type":155,"value":5591},{"type":150,"tag":417,"props":42043,"children":42044},{},[42045],{"type":155,"value":12954},{"type":150,"tag":411,"props":42047,"children":42048},{},[42049],{"type":155,"value":206},{"type":150,"tag":223,"props":42051,"children":42052},{"encoding":225},[42053],{"type":155,"value":42054},"N \\to 0",{"type":150,"tag":169,"props":42056,"children":42058},{"className":42057,"ariaHidden":214},[232],[42059,42085],{"type":150,"tag":169,"props":42060,"children":42062},{"className":42061},[237],[42063,42067,42072,42076,42081],{"type":150,"tag":169,"props":42064,"children":42066},{"className":42065,"style":5854},[242],[],{"type":150,"tag":169,"props":42068,"children":42070},{"className":42069,"style":5780},[248,489],[42071],{"type":155,"value":5591},{"type":150,"tag":169,"props":42073,"children":42075},{"className":42074,"style":839},[558],[],{"type":150,"tag":169,"props":42077,"children":42079},{"className":42078},[844],[42080],{"type":155,"value":12954},{"type":150,"tag":169,"props":42082,"children":42084},{"className":42083,"style":839},[558],[],{"type":150,"tag":169,"props":42086,"children":42088},{"className":42087},[237],[42089,42093],{"type":150,"tag":169,"props":42090,"children":42092},{"className":42091,"style":5186},[242],[],{"type":150,"tag":169,"props":42094,"children":42096},{"className":42095},[248],[42097],{"type":155,"value":206},{"type":155,"value":42099}," 时 ",{"type":150,"tag":169,"props":42101,"children":42103},{"className":42102},[177],[42104,42138],{"type":150,"tag":169,"props":42105,"children":42107},{"className":42106},[182],[42108],{"type":150,"tag":185,"props":42109,"children":42110},{"xmlns":187},[42111],{"type":150,"tag":191,"props":42112,"children":42113},{},[42114,42133],{"type":150,"tag":195,"props":42115,"children":42116},{},[42117,42121,42125,42129],{"type":150,"tag":217,"props":42118,"children":42119},{},[42120],{"type":155,"value":25141},{"type":150,"tag":417,"props":42122,"children":42123},{},[42124],{"type":155,"value":12954},{"type":150,"tag":411,"props":42126,"children":42127},{},[42128],{"type":155,"value":22571},{"type":150,"tag":405,"props":42130,"children":42131},{},[42132],{"type":155,"value":24918},{"type":150,"tag":223,"props":42134,"children":42135},{"encoding":225},[42136],{"type":155,"value":42137},"\\text{AI} \\to 4B",{"type":150,"tag":169,"props":42139,"children":42141},{"className":42140,"ariaHidden":214},[232],[42142,42172],{"type":150,"tag":169,"props":42143,"children":42145},{"className":42144},[237],[42146,42150,42159,42163,42168],{"type":150,"tag":169,"props":42147,"children":42149},{"className":42148,"style":5854},[242],[],{"type":150,"tag":169,"props":42151,"children":42153},{"className":42152},[248,155],[42154],{"type":150,"tag":169,"props":42155,"children":42157},{"className":42156},[248],[42158],{"type":155,"value":25141},{"type":150,"tag":169,"props":42160,"children":42162},{"className":42161,"style":839},[558],[],{"type":150,"tag":169,"props":42164,"children":42166},{"className":42165},[844],[42167],{"type":155,"value":12954},{"type":150,"tag":169,"props":42169,"children":42171},{"className":42170,"style":839},[558],[],{"type":150,"tag":169,"props":42173,"children":42175},{"className":42174},[237],[42176,42180,42185],{"type":150,"tag":169,"props":42177,"children":42179},{"className":42178,"style":5854},[242],[],{"type":150,"tag":169,"props":42181,"children":42183},{"className":42182},[248],[42184],{"type":155,"value":22571},{"type":150,"tag":169,"props":42186,"children":42188},{"className":42187,"style":27238},[248,489],[42189],{"type":155,"value":24918},{"type":155,"value":42191},"，所以 batch 要超过 280 才有可能碰到那条线。",{"type":150,"tag":169,"props":42193,"children":42195},{"className":42194},[177],[42196,42225],{"type":150,"tag":169,"props":42197,"children":42199},{"className":42198},[182],[42200],{"type":150,"tag":185,"props":42201,"children":42202},{"xmlns":187},[42203],{"type":150,"tag":191,"props":42204,"children":42205},{},[42206,42221],{"type":150,"tag":195,"props":42207,"children":42208},{},[42209,42213,42217],{"type":150,"tag":405,"props":42210,"children":42211},{},[42212],{"type":155,"value":24918},{"type":150,"tag":417,"props":42214,"children":42215},{},[42216],{"type":155,"value":1045},{"type":150,"tag":411,"props":42218,"children":42219},{},[42220],{"type":155,"value":20836},{"type":150,"tag":223,"props":42222,"children":42223},{"encoding":225},[42224],{"type":155,"value":28548},{"type":150,"tag":169,"props":42226,"children":42228},{"className":42227,"ariaHidden":214},[232],[42229,42255],{"type":150,"tag":169,"props":42230,"children":42232},{"className":42231},[237],[42233,42237,42242,42246,42251],{"type":150,"tag":169,"props":42234,"children":42236},{"className":42235,"style":5854},[242],[],{"type":150,"tag":169,"props":42238,"children":42240},{"className":42239,"style":27238},[248,489],[42241],{"type":155,"value":24918},{"type":150,"tag":169,"props":42243,"children":42245},{"className":42244,"style":839},[558],[],{"type":150,"tag":169,"props":42247,"children":42249},{"className":42248},[844],[42250],{"type":155,"value":1045},{"type":150,"tag":169,"props":42252,"children":42254},{"className":42253,"style":839},[558],[],{"type":150,"tag":169,"props":42256,"children":42258},{"className":42257},[237],[42259,42263],{"type":150,"tag":169,"props":42260,"children":42262},{"className":42261,"style":5186},[242],[],{"type":150,"tag":169,"props":42264,"children":42266},{"className":42265},[248],[42267],{"type":155,"value":20836},{"type":155,"value":42269}," 的极限是 256，连门槛都摸不到。",{"type":150,"tag":151,"props":42271,"children":42272},{},[42273,42315,42317,42393,42394,42529],{"type":150,"tag":169,"props":42274,"children":42276},{"className":42275},[177],[42277,42298],{"type":150,"tag":169,"props":42278,"children":42280},{"className":42279},[182],[42281],{"type":150,"tag":185,"props":42282,"children":42283},{"xmlns":187},[42284],{"type":150,"tag":191,"props":42285,"children":42286},{},[42287,42294],{"type":150,"tag":195,"props":42288,"children":42289},{},[42290],{"type":150,"tag":405,"props":42291,"children":42292},{},[42293],{"type":155,"value":5591},{"type":150,"tag":223,"props":42295,"children":42296},{"encoding":225},[42297],{"type":155,"value":5591},{"type":150,"tag":169,"props":42299,"children":42301},{"className":42300,"ariaHidden":214},[232],[42302],{"type":150,"tag":169,"props":42303,"children":42305},{"className":42304},[237],[42306,42310],{"type":150,"tag":169,"props":42307,"children":42309},{"className":42308,"style":5854},[242],[],{"type":150,"tag":169,"props":42311,"children":42313},{"className":42312,"style":5780},[248,489],[42314],{"type":155,"value":5591},{"type":155,"value":42316}," 一大就更没戏。",{"type":150,"tag":169,"props":42318,"children":42320},{"className":42319},[177],[42321,42350],{"type":150,"tag":169,"props":42322,"children":42324},{"className":42323},[182],[42325],{"type":150,"tag":185,"props":42326,"children":42327},{"xmlns":187},[42328],{"type":150,"tag":191,"props":42329,"children":42330},{},[42331,42346],{"type":150,"tag":195,"props":42332,"children":42333},{},[42334,42338,42342],{"type":150,"tag":405,"props":42335,"children":42336},{},[42337],{"type":155,"value":24918},{"type":150,"tag":417,"props":42339,"children":42340},{},[42341],{"type":155,"value":12954},{"type":150,"tag":405,"props":42343,"children":42344},{"mathvariant":1796},[42345],{"type":155,"value":26346},{"type":150,"tag":223,"props":42347,"children":42348},{"encoding":225},[42349],{"type":155,"value":28708},{"type":150,"tag":169,"props":42351,"children":42353},{"className":42352,"ariaHidden":214},[232],[42354,42380],{"type":150,"tag":169,"props":42355,"children":42357},{"className":42356},[237],[42358,42362,42367,42371,42376],{"type":150,"tag":169,"props":42359,"children":42361},{"className":42360,"style":5854},[242],[],{"type":150,"tag":169,"props":42363,"children":42365},{"className":42364,"style":27238},[248,489],[42366],{"type":155,"value":24918},{"type":150,"tag":169,"props":42368,"children":42370},{"className":42369,"style":839},[558],[],{"type":150,"tag":169,"props":42372,"children":42374},{"className":42373},[844],[42375],{"type":155,"value":12954},{"type":150,"tag":169,"props":42377,"children":42379},{"className":42378,"style":839},[558],[],{"type":150,"tag":169,"props":42381,"children":42383},{"className":42382},[237],[42384,42388],{"type":150,"tag":169,"props":42385,"children":42387},{"className":42386,"style":1094},[242],[],{"type":150,"tag":169,"props":42389,"children":42391},{"className":42390},[248],[42392],{"type":155,"value":26346},{"type":155,"value":42099},{"type":150,"tag":169,"props":42395,"children":42397},{"className":42396},[177],[42398,42452],{"type":150,"tag":169,"props":42399,"children":42401},{"className":42400},[182],[42402],{"type":150,"tag":185,"props":42403,"children":42404},{"xmlns":187},[42405],{"type":150,"tag":191,"props":42406,"children":42407},{},[42408,42447],{"type":150,"tag":195,"props":42409,"children":42410},{},[42411,42415,42419,42423,42427,42431,42435,42439,42443],{"type":150,"tag":217,"props":42412,"children":42413},{},[42414],{"type":155,"value":25141},{"type":150,"tag":417,"props":42416,"children":42417},{},[42418],{"type":155,"value":12954},{"type":150,"tag":411,"props":42420,"children":42421},{},[42422],{"type":155,"value":433},{"type":150,"tag":405,"props":42424,"children":42425},{},[42426],{"type":155,"value":25172},{"type":150,"tag":405,"props":42428,"children":42429},{"mathvariant":1796},[42430],{"type":155,"value":4914},{"type":150,"tag":417,"props":42432,"children":42433},{"stretchy":207},[42434],{"type":155,"value":972},{"type":150,"tag":411,"props":42436,"children":42437},{},[42438],{"type":155,"value":39544},{"type":150,"tag":405,"props":42440,"children":42441},{},[42442],{"type":155,"value":5591},{"type":150,"tag":417,"props":42444,"children":42445},{"stretchy":207},[42446],{"type":155,"value":982},{"type":150,"tag":223,"props":42448,"children":42449},{"encoding":225},[42450],{"type":155,"value":42451},"\\text{AI} \\to 2P \u002F (890N)",{"type":150,"tag":169,"props":42453,"children":42455},{"className":42454,"ariaHidden":214},[232],[42456,42486],{"type":150,"tag":169,"props":42457,"children":42459},{"className":42458},[237],[42460,42464,42473,42477,42482],{"type":150,"tag":169,"props":42461,"children":42463},{"className":42462,"style":5854},[242],[],{"type":150,"tag":169,"props":42465,"children":42467},{"className":42466},[248,155],[42468],{"type":150,"tag":169,"props":42469,"children":42471},{"className":42470},[248],[42472],{"type":155,"value":25141},{"type":150,"tag":169,"props":42474,"children":42476},{"className":42475,"style":839},[558],[],{"type":150,"tag":169,"props":42478,"children":42480},{"className":42479},[844],[42481],{"type":155,"value":12954},{"type":150,"tag":169,"props":42483,"children":42485},{"className":42484,"style":839},[558],[],{"type":150,"tag":169,"props":42487,"children":42489},{"className":42488},[237],[42490,42494,42499,42504,42509,42514,42519,42524],{"type":150,"tag":169,"props":42491,"children":42493},{"className":42492,"style":999},[242],[],{"type":150,"tag":169,"props":42495,"children":42497},{"className":42496},[248],[42498],{"type":155,"value":433},{"type":150,"tag":169,"props":42500,"children":42502},{"className":42501,"style":3869},[248,489],[42503],{"type":155,"value":25172},{"type":150,"tag":169,"props":42505,"children":42507},{"className":42506},[248],[42508],{"type":155,"value":4914},{"type":150,"tag":169,"props":42510,"children":42512},{"className":42511},[1004],[42513],{"type":155,"value":972},{"type":150,"tag":169,"props":42515,"children":42517},{"className":42516},[248],[42518],{"type":155,"value":39544},{"type":150,"tag":169,"props":42520,"children":42522},{"className":42521,"style":5780},[248,489],[42523],{"type":155,"value":5591},{"type":150,"tag":169,"props":42525,"children":42527},{"className":42526},[1015],[42528],{"type":155,"value":982},{"type":155,"value":42530},"，1M context 下只有 34，离 1125 差 33 倍。因为 KV 太小，把权重完全摊掉之后剩下的那点 KV 依然喂不饱算力。",{"type":150,"tag":151,"props":42532,"children":42533},{},[42534,42536,42612,42614,42656],{"type":155,"value":42535},"结论比 MLA 那次更极端。MLA 至少留了一个\"大 batch 加短 context 会翻到 compute bound\"的窗口，而且那个窗口落在 4k，恰好是它收益最小的区间。ds4.1f 的窗口更窄：要 ",{"type":150,"tag":169,"props":42537,"children":42539},{"className":42538},[177],[42540,42569],{"type":150,"tag":169,"props":42541,"children":42543},{"className":42542},[182],[42544],{"type":150,"tag":185,"props":42545,"children":42546},{"xmlns":187},[42547],{"type":150,"tag":191,"props":42548,"children":42549},{},[42550,42565],{"type":150,"tag":195,"props":42551,"children":42552},{},[42553,42557,42561],{"type":150,"tag":405,"props":42554,"children":42555},{},[42556],{"type":155,"value":24918},{"type":150,"tag":417,"props":42558,"children":42559},{},[42560],{"type":155,"value":41966},{"type":150,"tag":411,"props":42562,"children":42563},{},[42564],{"type":155,"value":20981},{"type":150,"tag":223,"props":42566,"children":42567},{"encoding":225},[42568],{"type":155,"value":41975},{"type":150,"tag":169,"props":42570,"children":42572},{"className":42571,"ariaHidden":214},[232],[42573,42599],{"type":150,"tag":169,"props":42574,"children":42576},{"className":42575},[237],[42577,42581,42586,42590,42595],{"type":150,"tag":169,"props":42578,"children":42580},{"className":42579,"style":41987},[242],[],{"type":150,"tag":169,"props":42582,"children":42584},{"className":42583,"style":27238},[248,489],[42585],{"type":155,"value":24918},{"type":150,"tag":169,"props":42587,"children":42589},{"className":42588,"style":839},[558],[],{"type":150,"tag":169,"props":42591,"children":42593},{"className":42592},[844],[42594],{"type":155,"value":41966},{"type":150,"tag":169,"props":42596,"children":42598},{"className":42597,"style":839},[558],[],{"type":150,"tag":169,"props":42600,"children":42602},{"className":42601},[237],[42603,42607],{"type":150,"tag":169,"props":42604,"children":42606},{"className":42605,"style":5186},[242],[],{"type":150,"tag":169,"props":42608,"children":42610},{"className":42609},[248],[42611],{"type":155,"value":20981},{"type":155,"value":42613}," 且 ",{"type":150,"tag":169,"props":42615,"children":42617},{"className":42616},[177],[42618,42639],{"type":150,"tag":169,"props":42619,"children":42621},{"className":42620},[182],[42622],{"type":150,"tag":185,"props":42623,"children":42624},{"xmlns":187},[42625],{"type":150,"tag":191,"props":42626,"children":42627},{},[42628,42635],{"type":150,"tag":195,"props":42629,"children":42630},{},[42631],{"type":150,"tag":405,"props":42632,"children":42633},{},[42634],{"type":155,"value":5591},{"type":150,"tag":223,"props":42636,"children":42637},{"encoding":225},[42638],{"type":155,"value":5591},{"type":150,"tag":169,"props":42640,"children":42642},{"className":42641,"ariaHidden":214},[232],[42643],{"type":150,"tag":169,"props":42644,"children":42646},{"className":42645},[237],[42647,42651],{"type":150,"tag":169,"props":42648,"children":42650},{"className":42649,"style":5854},[242],[],{"type":150,"tag":169,"props":42652,"children":42654},{"className":42653,"style":5780},[248,489],[42655],{"type":155,"value":5591},{"type":155,"value":42657}," 压到 1.4 万以内，才算勉强碰到算力那一侧。",{"type":150,"tag":151,"props":42659,"children":42660},{},[42661],{"type":155,"value":42662},"再算一遍吞吐。每步耗时取带宽和算力里较大的那个：",{"type":150,"tag":22368,"props":42664,"children":42665},{},[42666,42732],{"type":150,"tag":22372,"props":42667,"children":42668},{},[42669],{"type":150,"tag":22376,"props":42670,"children":42671},{},[42672,42717,42722,42727],{"type":150,"tag":22380,"props":42673,"children":42674},{},[42675],{"type":150,"tag":169,"props":42676,"children":42678},{"className":42677},[177],[42679,42700],{"type":150,"tag":169,"props":42680,"children":42682},{"className":42681},[182],[42683],{"type":150,"tag":185,"props":42684,"children":42685},{"xmlns":187},[42686],{"type":150,"tag":191,"props":42687,"children":42688},{},[42689,42696],{"type":150,"tag":195,"props":42690,"children":42691},{},[42692],{"type":150,"tag":405,"props":42693,"children":42694},{},[42695],{"type":155,"value":24918},{"type":150,"tag":223,"props":42697,"children":42698},{"encoding":225},[42699],{"type":155,"value":24918},{"type":150,"tag":169,"props":42701,"children":42703},{"className":42702,"ariaHidden":214},[232],[42704],{"type":150,"tag":169,"props":42705,"children":42707},{"className":42706},[237],[42708,42712],{"type":150,"tag":169,"props":42709,"children":42711},{"className":42710,"style":5854},[242],[],{"type":150,"tag":169,"props":42713,"children":42715},{"className":42714,"style":27238},[248,489],[42716],{"type":155,"value":24918},{"type":150,"tag":22380,"props":42718,"children":42719},{},[42720],{"type":155,"value":42721},"1M：单序列 tok\u002Fs",{"type":150,"tag":22380,"props":42723,"children":42724},{},[42725],{"type":155,"value":42726},"1M：总吞吐",{"type":150,"tag":22380,"props":42728,"children":42729},{},[42730],{"type":155,"value":42731},"32k：总吞吐",{"type":150,"tag":22394,"props":42733,"children":42734},{},[42735,42754,42776,42798,42820,42841],{"type":150,"tag":22376,"props":42736,"children":42737},{},[42738,42742,42746,42750],{"type":150,"tag":22401,"props":42739,"children":42740},{},[42741],{"type":155,"value":415},{"type":150,"tag":22401,"props":42743,"children":42744},{},[42745],{"type":155,"value":40669},{"type":150,"tag":22401,"props":42747,"children":42748},{},[42749],{"type":155,"value":40669},{"type":150,"tag":22401,"props":42751,"children":42752},{},[42753],{"type":155,"value":40607},{"type":150,"tag":22376,"props":42755,"children":42756},{},[42757,42761,42766,42771],{"type":150,"tag":22401,"props":42758,"children":42759},{},[42760],{"type":155,"value":9389},{"type":150,"tag":22401,"props":42762,"children":42763},{},[42764],{"type":155,"value":42765},"517",{"type":150,"tag":22401,"props":42767,"children":42768},{},[42769],{"type":155,"value":42770},"4,138",{"type":150,"tag":22401,"props":42772,"children":42773},{},[42774],{"type":155,"value":42775},"7,773",{"type":150,"tag":22376,"props":42777,"children":42778},{},[42779,42783,42788,42793],{"type":150,"tag":22401,"props":42780,"children":42781},{},[42782],{"type":155,"value":36879},{"type":150,"tag":22401,"props":42784,"children":42785},{},[42786],{"type":155,"value":42787},"211",{"type":150,"tag":22401,"props":42789,"children":42790},{},[42791],{"type":155,"value":42792},"6,761",{"type":150,"tag":22401,"props":42794,"children":42795},{},[42796],{"type":155,"value":42797},"28,657",{"type":150,"tag":22376,"props":42799,"children":42800},{},[42801,42805,42810,42815],{"type":150,"tag":22401,"props":42802,"children":42803},{},[42804],{"type":155,"value":6045},{"type":150,"tag":22401,"props":42806,"children":42807},{},[42808],{"type":155,"value":42809},"63",{"type":150,"tag":22401,"props":42811,"children":42812},{},[42813],{"type":155,"value":42814},"8,034",{"type":150,"tag":22401,"props":42816,"children":42817},{},[42818],{"type":155,"value":42819},"87,276",{"type":150,"tag":22376,"props":42821,"children":42822},{},[42823,42827,42831,42836],{"type":150,"tag":22401,"props":42824,"children":42825},{},[42826],{"type":155,"value":20981},{"type":150,"tag":22401,"props":42828,"children":42829},{},[42830],{"type":155,"value":36853},{"type":150,"tag":22401,"props":42832,"children":42833},{},[42834],{"type":155,"value":42835},"8,431",{"type":150,"tag":22401,"props":42837,"children":42838},{},[42839],{"type":155,"value":42840},"178,617",{"type":150,"tag":22376,"props":42842,"children":42843},{},[42844,42912,42917,42922],{"type":150,"tag":22401,"props":42845,"children":42846},{},[42847],{"type":150,"tag":169,"props":42848,"children":42850},{"className":42849},[177],[42851,42877],{"type":150,"tag":169,"props":42852,"children":42854},{"className":42853},[182],[42855],{"type":150,"tag":185,"props":42856,"children":42857},{"xmlns":187},[42858],{"type":150,"tag":191,"props":42859,"children":42860},{},[42861,42872],{"type":150,"tag":195,"props":42862,"children":42863},{},[42864,42868],{"type":150,"tag":417,"props":42865,"children":42866},{},[42867],{"type":155,"value":12954},{"type":150,"tag":405,"props":42869,"children":42870},{"mathvariant":1796},[42871],{"type":155,"value":26346},{"type":150,"tag":223,"props":42873,"children":42874},{"encoding":225},[42875],{"type":155,"value":42876},"\\to \\infty",{"type":150,"tag":169,"props":42878,"children":42880},{"className":42879,"ariaHidden":214},[232],[42881,42899],{"type":150,"tag":169,"props":42882,"children":42884},{"className":42883},[237],[42885,42890,42895],{"type":150,"tag":169,"props":42886,"children":42889},{"className":42887,"style":42888},[242],"height:0.3669em;",[],{"type":150,"tag":169,"props":42891,"children":42893},{"className":42892},[844],[42894],{"type":155,"value":12954},{"type":150,"tag":169,"props":42896,"children":42898},{"className":42897,"style":839},[558],[],{"type":150,"tag":169,"props":42900,"children":42902},{"className":42901},[237],[42903,42907],{"type":150,"tag":169,"props":42904,"children":42906},{"className":42905,"style":1094},[242],[],{"type":150,"tag":169,"props":42908,"children":42910},{"className":42909},[248],[42911],{"type":155,"value":26346},{"type":150,"tag":22401,"props":42913,"children":42914},{},[42915],{"type":155,"value":42916},"趋于 0",{"type":150,"tag":22401,"props":42918,"children":42919},{},[42920],{"type":155,"value":42921},"8,572",{"type":150,"tag":22401,"props":42923,"children":42924},{},[42925],{"type":155,"value":42926},"274,315",{"type":150,"tag":151,"props":42928,"children":42929},{},[42930],{"type":150,"tag":158,"props":42931,"children":42932},{},[42933],{"type":155,"value":42934},"总吞吐有上限，因为它最终被 KV 的带宽锁住：",{"type":150,"tag":169,"props":42936,"children":42938},{"className":42937},[172],[42939],{"type":150,"tag":169,"props":42940,"children":42942},{"className":42941},[177],[42943,43015],{"type":150,"tag":169,"props":42944,"children":42946},{"className":42945},[182],[42947],{"type":150,"tag":185,"props":42948,"children":42949},{"xmlns":187,"display":188},[42950],{"type":150,"tag":191,"props":42951,"children":42952},{},[42953,43010],{"type":150,"tag":195,"props":42954,"children":42955},{},[42956,42973,42977],{"type":150,"tag":401,"props":42957,"children":42958},{},[42959,42964,42969],{"type":150,"tag":217,"props":42960,"children":42961},{},[42962],{"type":155,"value":42963},"吞吐",{"type":150,"tag":405,"props":42965,"children":42966},{},[42967],{"type":155,"value":42968},"max",{"type":150,"tag":417,"props":42970,"children":42971},{},[42972],{"type":155,"value":1991},{"type":150,"tag":417,"props":42974,"children":42975},{},[42976],{"type":155,"value":12954},{"type":150,"tag":1952,"props":42978,"children":42979},{},[42980,42991],{"type":150,"tag":195,"props":42981,"children":42982},{},[42983,42987],{"type":150,"tag":411,"props":42984,"children":42985},{},[42986],{"type":155,"value":9389},{"type":150,"tag":217,"props":42988,"children":42989},{},[42990],{"type":155,"value":23629},{"type":150,"tag":195,"props":42992,"children":42993},{},[42994,42998,43002,43006],{"type":150,"tag":411,"props":42995,"children":42996},{},[42997],{"type":155,"value":39544},{"type":150,"tag":217,"props":42999,"children":43000},{},[43001],{"type":155,"value":40732},{"type":150,"tag":417,"props":43003,"children":43004},{},[43005],{"type":155,"value":5554},{"type":150,"tag":405,"props":43007,"children":43008},{},[43009],{"type":155,"value":5591},{"type":150,"tag":223,"props":43011,"children":43012},{"encoding":225},[43013],{"type":155,"value":43014},"\\text{吞吐}_{\\max} \\to \\frac{8\\ \\text{TB\u002Fs}}{890\\ \\text{B} \\times N}",{"type":150,"tag":169,"props":43016,"children":43018},{"className":43017,"ariaHidden":214},[232],[43019,43121],{"type":150,"tag":169,"props":43020,"children":43022},{"className":43021},[237],[43023,43027,43108,43112,43117],{"type":150,"tag":169,"props":43024,"children":43026},{"className":43025,"style":6364},[242],[],{"type":150,"tag":169,"props":43028,"children":43030},{"className":43029},[248],[43031,43040],{"type":150,"tag":169,"props":43032,"children":43034},{"className":43033},[248,155],[43035],{"type":150,"tag":169,"props":43036,"children":43038},{"className":43037},[248,297],[43039],{"type":155,"value":42963},{"type":150,"tag":169,"props":43041,"children":43043},{"className":43042},[495],[43044],{"type":150,"tag":169,"props":43045,"children":43047},{"className":43046},[253,254],[43048,43097],{"type":150,"tag":169,"props":43049,"children":43051},{"className":43050},[259],[43052,43092],{"type":150,"tag":169,"props":43053,"children":43055},{"className":43054,"style":5039},[264],[43056],{"type":150,"tag":169,"props":43057,"children":43059},{"style":43058},"top:-2.55em;margin-right:0.05em;",[43060,43064],{"type":150,"tag":169,"props":43061,"children":43063},{"className":43062,"style":517},[274],[],{"type":150,"tag":169,"props":43065,"children":43067},{"className":43066},[522,523,524,525],[43068],{"type":150,"tag":169,"props":43069,"children":43071},{"className":43070},[248,525],[43072],{"type":150,"tag":169,"props":43073,"children":43075},{"className":43074},[2695,525],[43076,43082,43087],{"type":150,"tag":169,"props":43077,"children":43079},{"className":43078},[525],[43080],{"type":155,"value":43081},"m",{"type":150,"tag":169,"props":43083,"children":43085},{"className":43084},[525],[43086],{"type":155,"value":9850},{"type":150,"tag":169,"props":43088,"children":43090},{"className":43089},[525],[43091],{"type":155,"value":409},{"type":150,"tag":169,"props":43093,"children":43095},{"className":43094},[355],[43096],{"type":155,"value":358},{"type":150,"tag":169,"props":43098,"children":43100},{"className":43099},[259],[43101],{"type":150,"tag":169,"props":43102,"children":43104},{"className":43103,"style":544},[264],[43105],{"type":150,"tag":169,"props":43106,"children":43107},{},[],{"type":150,"tag":169,"props":43109,"children":43111},{"className":43110,"style":839},[558],[],{"type":150,"tag":169,"props":43113,"children":43115},{"className":43114},[844],[43116],{"type":155,"value":12954},{"type":150,"tag":169,"props":43118,"children":43120},{"className":43119,"style":839},[558],[],{"type":150,"tag":169,"props":43122,"children":43124},{"className":43123},[237],[43125,43130],{"type":150,"tag":169,"props":43126,"children":43129},{"className":43127,"style":43128},[242],"height:2.1963em;vertical-align:-0.7693em;",[],{"type":150,"tag":169,"props":43131,"children":43133},{"className":43132},[248],[43134,43138,43259],{"type":150,"tag":169,"props":43135,"children":43137},{"className":43136},[1004,2660],[],{"type":150,"tag":169,"props":43139,"children":43141},{"className":43140},[1952],[43142],{"type":150,"tag":169,"props":43143,"children":43145},{"className":43144},[253,254],[43146,43248],{"type":150,"tag":169,"props":43147,"children":43149},{"className":43148},[259],[43150,43243],{"type":150,"tag":169,"props":43151,"children":43153},{"className":43152,"style":27482},[264],[43154,43202,43213],{"type":150,"tag":169,"props":43155,"children":43156},{"style":8422},[43157,43161],{"type":150,"tag":169,"props":43158,"children":43160},{"className":43159,"style":8427},[274],[],{"type":150,"tag":169,"props":43162,"children":43164},{"className":43163},[248],[43165,43170,43175,43184,43188,43193,43197],{"type":150,"tag":169,"props":43166,"children":43168},{"className":43167},[248],[43169],{"type":155,"value":39544},{"type":150,"tag":169,"props":43171,"children":43173},{"className":43172},[558],[43174],{"type":155,"value":8763},{"type":150,"tag":169,"props":43176,"children":43178},{"className":43177},[248,155],[43179],{"type":150,"tag":169,"props":43180,"children":43182},{"className":43181},[248],[43183],{"type":155,"value":24918},{"type":150,"tag":169,"props":43185,"children":43187},{"className":43186,"style":5167},[558],[],{"type":150,"tag":169,"props":43189,"children":43191},{"className":43190},[5172],[43192],{"type":155,"value":5554},{"type":150,"tag":169,"props":43194,"children":43196},{"className":43195,"style":5167},[558],[],{"type":150,"tag":169,"props":43198,"children":43200},{"className":43199,"style":5780},[248,489],[43201],{"type":155,"value":5591},{"type":150,"tag":169,"props":43203,"children":43204},{"style":8440},[43205,43209],{"type":150,"tag":169,"props":43206,"children":43208},{"className":43207,"style":8427},[274],[],{"type":150,"tag":169,"props":43210,"children":43212},{"className":43211,"style":2982},[2981],[],{"type":150,"tag":169,"props":43214,"children":43215},{"style":8452},[43216,43220],{"type":150,"tag":169,"props":43217,"children":43219},{"className":43218,"style":8427},[274],[],{"type":150,"tag":169,"props":43221,"children":43223},{"className":43222},[248],[43224,43229,43234],{"type":150,"tag":169,"props":43225,"children":43227},{"className":43226},[248],[43228],{"type":155,"value":9389},{"type":150,"tag":169,"props":43230,"children":43232},{"className":43231},[558],[43233],{"type":155,"value":8763},{"type":150,"tag":169,"props":43235,"children":43237},{"className":43236},[248,155],[43238],{"type":150,"tag":169,"props":43239,"children":43241},{"className":43240},[248],[43242],{"type":155,"value":23717},{"type":150,"tag":169,"props":43244,"children":43246},{"className":43245},[355],[43247],{"type":155,"value":358},{"type":150,"tag":169,"props":43249,"children":43251},{"className":43250},[259],[43252],{"type":150,"tag":169,"props":43253,"children":43255},{"className":43254,"style":26239},[264],[43256],{"type":150,"tag":169,"props":43257,"children":43258},{},[],{"type":150,"tag":169,"props":43260,"children":43262},{"className":43261},[1015,2660],[],{"type":150,"tag":22368,"props":43264,"children":43265},{},[43266,43333],{"type":150,"tag":22372,"props":43267,"children":43268},{},[43269],{"type":150,"tag":22376,"props":43270,"children":43271},{},[43272,43317,43321,43325,43329],{"type":150,"tag":22380,"props":43273,"children":43274},{},[43275],{"type":150,"tag":169,"props":43276,"children":43278},{"className":43277},[177],[43279,43300],{"type":150,"tag":169,"props":43280,"children":43282},{"className":43281},[182],[43283],{"type":150,"tag":185,"props":43284,"children":43285},{"xmlns":187},[43286],{"type":150,"tag":191,"props":43287,"children":43288},{},[43289,43296],{"type":150,"tag":195,"props":43290,"children":43291},{},[43292],{"type":150,"tag":405,"props":43293,"children":43294},{},[43295],{"type":155,"value":5591},{"type":150,"tag":223,"props":43297,"children":43298},{"encoding":225},[43299],{"type":155,"value":5591},{"type":150,"tag":169,"props":43301,"children":43303},{"className":43302,"ariaHidden":214},[232],[43304],{"type":150,"tag":169,"props":43305,"children":43307},{"className":43306},[237],[43308,43312],{"type":150,"tag":169,"props":43309,"children":43311},{"className":43310,"style":5854},[242],[],{"type":150,"tag":169,"props":43313,"children":43315},{"className":43314,"style":5780},[248,489],[43316],{"type":155,"value":5591},{"type":150,"tag":22380,"props":43318,"children":43319},{},[43320],{"type":155,"value":27081},{"type":150,"tag":22380,"props":43322,"children":43323},{},[43324],{"type":155,"value":27109},{"type":150,"tag":22380,"props":43326,"children":43327},{},[43328],{"type":155,"value":27136},{"type":150,"tag":22380,"props":43330,"children":43331},{},[43332],{"type":155,"value":27163},{"type":150,"tag":22394,"props":43334,"children":43335},{},[43336],{"type":150,"tag":22376,"props":43337,"children":43338},{},[43339,43344,43349,43354,43359],{"type":150,"tag":22401,"props":43340,"children":43341},{},[43342],{"type":155,"value":43343},"吞吐上限",{"type":150,"tag":22401,"props":43345,"children":43346},{},[43347],{"type":155,"value":43348},"281k",{"type":150,"tag":22401,"props":43350,"children":43351},{},[43352],{"type":155,"value":43353},"274k",{"type":150,"tag":22401,"props":43355,"children":43356},{},[43357],{"type":155,"value":43358},"68.6k",{"type":150,"tag":22401,"props":43360,"children":43361},{},[43362],{"type":155,"value":43363},"8.6k",{"type":150,"tag":151,"props":43365,"children":43366},{},[43367,43369,43447],{"type":155,"value":43368},"4k 和 32k 那两档其实先被算力封顶（281k），再往上才是 KV 带宽封顶。分界在 ",{"type":150,"tag":169,"props":43370,"children":43372},{"className":43371},[177],[43373,43404],{"type":150,"tag":169,"props":43374,"children":43376},{"className":43375},[182],[43377],{"type":150,"tag":185,"props":43378,"children":43379},{"xmlns":187},[43380],{"type":150,"tag":191,"props":43381,"children":43382},{},[43383,43399],{"type":150,"tag":195,"props":43384,"children":43385},{},[43386,43390,43394],{"type":150,"tag":405,"props":43387,"children":43388},{},[43389],{"type":155,"value":5591},{"type":150,"tag":417,"props":43391,"children":43392},{},[43393],{"type":155,"value":6050},{"type":150,"tag":411,"props":43395,"children":43396},{},[43397],{"type":155,"value":43398},"3.2",{"type":150,"tag":223,"props":43400,"children":43401},{"encoding":225},[43402],{"type":155,"value":43403},"N \\approx 3.2",{"type":150,"tag":169,"props":43405,"children":43407},{"className":43406,"ariaHidden":214},[232],[43408,43434],{"type":150,"tag":169,"props":43409,"children":43411},{"className":43410},[237],[43412,43416,43421,43425,43430],{"type":150,"tag":169,"props":43413,"children":43415},{"className":43414,"style":5854},[242],[],{"type":150,"tag":169,"props":43417,"children":43419},{"className":43418,"style":5780},[248,489],[43420],{"type":155,"value":5591},{"type":150,"tag":169,"props":43422,"children":43424},{"className":43423,"style":839},[558],[],{"type":150,"tag":169,"props":43426,"children":43428},{"className":43427},[844],[43429],{"type":155,"value":6050},{"type":150,"tag":169,"props":43431,"children":43433},{"className":43432,"style":839},[558],[],{"type":150,"tag":169,"props":43435,"children":43437},{"className":43436},[237],[43438,43442],{"type":150,"tag":169,"props":43439,"children":43441},{"className":43440,"style":5186},[242],[],{"type":150,"tag":169,"props":43443,"children":43445},{"className":43444},[248],[43446],{"type":155,"value":43398},{"type":155,"value":43448}," 万，和前面 batch 表里算出来的是同一个数。",{"type":150,"tag":151,"props":43450,"children":43451},{},[43452,43457],{"type":150,"tag":158,"props":43453,"children":43454},{},[43455],{"type":155,"value":43456},"代价在延迟那一列。",{"type":155,"value":43458}," 同一个 1M 的请求，batch 从 1 开到 512，单序列速度从 896 掉到 16 token\u002Fs。长 context 上吞吐和交互速度是直接对立的：要吃满吞吐，就得牺牲单个用户的等待时间。",{"type":150,"tag":151,"props":43460,"children":43461},{},[43462],{"type":155,"value":43463},"反过来，如果要求单序列不低于 200 token\u002Fs，1M context 下 batch 只能开到 30 上下，单卡吞吐就是 6-7k token\u002Fs。这个量级和我实际跑 ds4.1f 的手感对得上。",{"type":150,"tag":151,"props":43465,"children":43466},{},[43467],{"type":155,"value":43468},"换成 TP4 部署，每卡的账是这样：",{"type":150,"tag":22368,"props":43470,"children":43471},{},[43472,43486],{"type":150,"tag":22372,"props":43473,"children":43474},{},[43475],{"type":150,"tag":22376,"props":43476,"children":43477},{},[43478,43481],{"type":150,"tag":22380,"props":43479,"children":43480},{},[],{"type":150,"tag":22380,"props":43482,"children":43483},{},[43484],{"type":155,"value":43485},"每卡",{"type":150,"tag":22394,"props":43487,"children":43488},{},[43489,43502,43515,43531,43547,43559],{"type":150,"tag":22376,"props":43490,"children":43491},{},[43492,43497],{"type":150,"tag":22401,"props":43493,"children":43494},{},[43495],{"type":155,"value":43496},"权重常驻显存（全部 552B 专家）",{"type":150,"tag":22401,"props":43498,"children":43499},{},[43500],{"type":155,"value":43501},"69 GB",{"type":150,"tag":22376,"props":43503,"children":43504},{},[43505,43510],{"type":150,"tag":22401,"props":43506,"children":43507},{},[43508],{"type":155,"value":43509},"留给 KV 的显存",{"type":150,"tag":22401,"props":43511,"children":43512},{},[43513],{"type":155,"value":43514},"111 GB",{"type":150,"tag":22376,"props":43516,"children":43517},{},[43518,43523],{"type":150,"tag":22401,"props":43519,"children":43520},{},[43521],{"type":155,"value":43522},"每步读的权重（16B activated 的四分之一）",{"type":150,"tag":22401,"props":43524,"children":43525},{},[43526],{"type":150,"tag":158,"props":43527,"children":43528},{},[43529],{"type":155,"value":43530},"2.00 GB",{"type":150,"tag":22376,"props":43532,"children":43533},{},[43534,43539],{"type":150,"tag":22401,"props":43535,"children":43536},{},[43537],{"type":155,"value":43538},"每步读的 KV（batch 1，1M context）",{"type":150,"tag":22401,"props":43540,"children":43541},{},[43542],{"type":150,"tag":158,"props":43543,"children":43544},{},[43545],{"type":155,"value":43546},"0.23 GB",{"type":150,"tag":22376,"props":43548,"children":43549},{},[43550,43554],{"type":150,"tag":22401,"props":43551,"children":43552},{},[43553],{"type":155,"value":40533},{"type":150,"tag":22401,"props":43555,"children":43556},{},[43557],{"type":155,"value":43558},"2.23 GB",{"type":150,"tag":22376,"props":43560,"children":43561},{},[43562,43567],{"type":150,"tag":22401,"props":43563,"children":43564},{},[43565],{"type":155,"value":43566},"每步带宽下限",{"type":150,"tag":22401,"props":43568,"children":43569},{},[43570],{"type":155,"value":43571},"0.28 ms",{"type":150,"tag":151,"props":43573,"children":43574},{},[43575],{"type":155,"value":43576},"要分清常驻和每步读，也要交代那个除以 4 是哪来的。",{"type":150,"tag":151,"props":43578,"children":43579},{},[43580,43585],{"type":150,"tag":158,"props":43581,"children":43582},{},[43583],{"type":155,"value":43584},"常驻",{"type":155,"value":43586},"：552B 个专家参数必须都在显存里。TP4 把每个权重矩阵切成四份、每卡持一份，所以是",{"type":150,"tag":169,"props":43588,"children":43590},{"className":43589},[172],[43591],{"type":150,"tag":169,"props":43592,"children":43594},{"className":43593},[177],[43595,43657],{"type":150,"tag":169,"props":43596,"children":43598},{"className":43597},[182],[43599],{"type":150,"tag":185,"props":43600,"children":43601},{"xmlns":187,"display":188},[43602],{"type":150,"tag":191,"props":43603,"children":43604},{},[43605,43652],{"type":150,"tag":195,"props":43606,"children":43607},{},[43608,43639,43643,43648],{"type":150,"tag":1952,"props":43609,"children":43610},{},[43611,43635],{"type":150,"tag":195,"props":43612,"children":43613},{},[43614,43619,43623,43627,43631],{"type":150,"tag":411,"props":43615,"children":43616},{},[43617],{"type":155,"value":43618},"552",{"type":150,"tag":217,"props":43620,"children":43621},{},[43622],{"type":155,"value":24918},{"type":150,"tag":417,"props":43624,"children":43625},{},[43626],{"type":155,"value":5554},{"type":150,"tag":411,"props":43628,"children":43629},{},[43630],{"type":155,"value":39390},{"type":150,"tag":217,"props":43632,"children":43633},{},[43634],{"type":155,"value":24931},{"type":150,"tag":411,"props":43636,"children":43637},{},[43638],{"type":155,"value":22571},{"type":150,"tag":417,"props":43640,"children":43641},{},[43642],{"type":155,"value":1045},{"type":150,"tag":411,"props":43644,"children":43645},{},[43646],{"type":155,"value":43647},"69",{"type":150,"tag":217,"props":43649,"children":43650},{},[43651],{"type":155,"value":29467},{"type":150,"tag":223,"props":43653,"children":43654},{"encoding":225},[43655],{"type":155,"value":43656},"\\frac{552\\text{B} \\times 0.5\\ \\text{字节}}{4} = 69\\ \\text{GB}",{"type":150,"tag":169,"props":43658,"children":43660},{"className":43659,"ariaHidden":214},[232],[43661,43810],{"type":150,"tag":169,"props":43662,"children":43664},{"className":43663},[237],[43665,43669,43797,43801,43806],{"type":150,"tag":169,"props":43666,"children":43668},{"className":43667,"style":31864},[242],[],{"type":150,"tag":169,"props":43670,"children":43672},{"className":43671},[248],[43673,43677,43793],{"type":150,"tag":169,"props":43674,"children":43676},{"className":43675},[1004,2660],[],{"type":150,"tag":169,"props":43678,"children":43680},{"className":43679},[1952],[43681],{"type":150,"tag":169,"props":43682,"children":43684},{"className":43683},[253,254],[43685,43782],{"type":150,"tag":169,"props":43686,"children":43688},{"className":43687},[259],[43689,43777],{"type":150,"tag":169,"props":43690,"children":43692},{"className":43691,"style":23685},[264],[43693,43709,43720],{"type":150,"tag":169,"props":43694,"children":43695},{"style":8422},[43696,43700],{"type":150,"tag":169,"props":43697,"children":43699},{"className":43698,"style":8427},[274],[],{"type":150,"tag":169,"props":43701,"children":43703},{"className":43702},[248],[43704],{"type":150,"tag":169,"props":43705,"children":43707},{"className":43706},[248],[43708],{"type":155,"value":22571},{"type":150,"tag":169,"props":43710,"children":43711},{"style":8440},[43712,43716],{"type":150,"tag":169,"props":43713,"children":43715},{"className":43714,"style":8427},[274],[],{"type":150,"tag":169,"props":43717,"children":43719},{"className":43718,"style":2982},[2981],[],{"type":150,"tag":169,"props":43721,"children":43722},{"style":8452},[43723,43727],{"type":150,"tag":169,"props":43724,"children":43726},{"className":43725,"style":8427},[274],[],{"type":150,"tag":169,"props":43728,"children":43730},{"className":43729},[248],[43731,43736,43745,43749,43754,43758,43763,43768],{"type":150,"tag":169,"props":43732,"children":43734},{"className":43733},[248],[43735],{"type":155,"value":43618},{"type":150,"tag":169,"props":43737,"children":43739},{"className":43738},[248,155],[43740],{"type":150,"tag":169,"props":43741,"children":43743},{"className":43742},[248],[43744],{"type":155,"value":24918},{"type":150,"tag":169,"props":43746,"children":43748},{"className":43747,"style":5167},[558],[],{"type":150,"tag":169,"props":43750,"children":43752},{"className":43751},[5172],[43753],{"type":155,"value":5554},{"type":150,"tag":169,"props":43755,"children":43757},{"className":43756,"style":5167},[558],[],{"type":150,"tag":169,"props":43759,"children":43761},{"className":43760},[248],[43762],{"type":155,"value":39390},{"type":150,"tag":169,"props":43764,"children":43766},{"className":43765},[558],[43767],{"type":155,"value":8763},{"type":150,"tag":169,"props":43769,"children":43771},{"className":43770},[248,155],[43772],{"type":150,"tag":169,"props":43773,"children":43775},{"className":43774},[248,297],[43776],{"type":155,"value":25038},{"type":150,"tag":169,"props":43778,"children":43780},{"className":43779},[355],[43781],{"type":155,"value":358},{"type":150,"tag":169,"props":43783,"children":43785},{"className":43784},[259],[43786],{"type":150,"tag":169,"props":43787,"children":43789},{"className":43788,"style":8484},[264],[43790],{"type":150,"tag":169,"props":43791,"children":43792},{},[],{"type":150,"tag":169,"props":43794,"children":43796},{"className":43795},[1015,2660],[],{"type":150,"tag":169,"props":43798,"children":43800},{"className":43799,"style":839},[558],[],{"type":150,"tag":169,"props":43802,"children":43804},{"className":43803},[844],[43805],{"type":155,"value":1045},{"type":150,"tag":169,"props":43807,"children":43809},{"className":43808,"style":839},[558],[],{"type":150,"tag":169,"props":43811,"children":43813},{"className":43812},[237],[43814,43818,43823,43828],{"type":150,"tag":169,"props":43815,"children":43817},{"className":43816,"style":5854},[242],[],{"type":150,"tag":169,"props":43819,"children":43821},{"className":43820},[248],[43822],{"type":155,"value":43647},{"type":150,"tag":169,"props":43824,"children":43826},{"className":43825},[558],[43827],{"type":155,"value":8763},{"type":150,"tag":169,"props":43829,"children":43831},{"className":43830},[248,155],[43832],{"type":150,"tag":169,"props":43833,"children":43835},{"className":43834},[248],[43836],{"type":155,"value":29613},{"type":150,"tag":151,"props":43838,"children":43839},{},[43840,43845],{"type":150,"tag":158,"props":43841,"children":43842},{},[43843],{"type":155,"value":43844},"每步读",{"type":155,"value":43846},"：一个 token 只激活其中 16B。整机每步要读",{"type":150,"tag":169,"props":43848,"children":43850},{"className":43849},[172],[43851],{"type":150,"tag":169,"props":43852,"children":43854},{"className":43853},[177],[43855,43904],{"type":150,"tag":169,"props":43856,"children":43858},{"className":43857},[182],[43859],{"type":150,"tag":185,"props":43860,"children":43861},{"xmlns":187,"display":188},[43862],{"type":150,"tag":191,"props":43863,"children":43864},{},[43865,43900],{"type":150,"tag":195,"props":43866,"children":43867},{},[43868,43872,43876,43880,43884,43888,43892,43896],{"type":150,"tag":411,"props":43869,"children":43870},{},[43871],{"type":155,"value":36853},{"type":150,"tag":217,"props":43873,"children":43874},{},[43875],{"type":155,"value":24918},{"type":150,"tag":417,"props":43877,"children":43878},{},[43879],{"type":155,"value":5554},{"type":150,"tag":411,"props":43881,"children":43882},{},[43883],{"type":155,"value":39390},{"type":150,"tag":217,"props":43885,"children":43886},{},[43887],{"type":155,"value":24931},{"type":150,"tag":417,"props":43889,"children":43890},{},[43891],{"type":155,"value":1045},{"type":150,"tag":411,"props":43893,"children":43894},{},[43895],{"type":155,"value":9389},{"type":150,"tag":217,"props":43897,"children":43898},{},[43899],{"type":155,"value":29467},{"type":150,"tag":223,"props":43901,"children":43902},{"encoding":225},[43903],{"type":155,"value":39411},{"type":150,"tag":169,"props":43905,"children":43907},{"className":43906,"ariaHidden":214},[232],[43908,43943,43983],{"type":150,"tag":169,"props":43909,"children":43911},{"className":43910},[237],[43912,43916,43921,43930,43934,43939],{"type":150,"tag":169,"props":43913,"children":43915},{"className":43914,"style":5644},[242],[],{"type":150,"tag":169,"props":43917,"children":43919},{"className":43918},[248],[43920],{"type":155,"value":36853},{"type":150,"tag":169,"props":43922,"children":43924},{"className":43923},[248,155],[43925],{"type":150,"tag":169,"props":43926,"children":43928},{"className":43927},[248],[43929],{"type":155,"value":24918},{"type":150,"tag":169,"props":43931,"children":43933},{"className":43932,"style":5167},[558],[],{"type":150,"tag":169,"props":43935,"children":43937},{"className":43936},[5172],[43938],{"type":155,"value":5554},{"type":150,"tag":169,"props":43940,"children":43942},{"className":43941,"style":5167},[558],[],{"type":150,"tag":169,"props":43944,"children":43946},{"className":43945},[237],[43947,43951,43956,43961,43970,43974,43979],{"type":150,"tag":169,"props":43948,"children":43950},{"className":43949,"style":5854},[242],[],{"type":150,"tag":169,"props":43952,"children":43954},{"className":43953},[248],[43955],{"type":155,"value":39390},{"type":150,"tag":169,"props":43957,"children":43959},{"className":43958},[558],[43960],{"type":155,"value":8763},{"type":150,"tag":169,"props":43962,"children":43964},{"className":43963},[248,155],[43965],{"type":150,"tag":169,"props":43966,"children":43968},{"className":43967},[248,297],[43969],{"type":155,"value":25038},{"type":150,"tag":169,"props":43971,"children":43973},{"className":43972,"style":839},[558],[],{"type":150,"tag":169,"props":43975,"children":43977},{"className":43976},[844],[43978],{"type":155,"value":1045},{"type":150,"tag":169,"props":43980,"children":43982},{"className":43981,"style":839},[558],[],{"type":150,"tag":169,"props":43984,"children":43986},{"className":43985},[237],[43987,43991,43996,44001],{"type":150,"tag":169,"props":43988,"children":43990},{"className":43989,"style":5854},[242],[],{"type":150,"tag":169,"props":43992,"children":43994},{"className":43993},[248],[43995],{"type":155,"value":9389},{"type":150,"tag":169,"props":43997,"children":43999},{"className":43998},[558],[44000],{"type":155,"value":8763},{"type":150,"tag":169,"props":44002,"children":44004},{"className":44003},[248,155],[44005],{"type":150,"tag":169,"props":44006,"children":44008},{"className":44007},[248],[44009],{"type":155,"value":29613},{"type":150,"tag":151,"props":44011,"children":44012},{},[44013],{"type":155,"value":44014},"而这 16B 激活参数同样散在四张卡上，各卡只读自己那一份，所以每卡读四分之一：",{"type":150,"tag":169,"props":44016,"children":44018},{"className":44017},[172],[44019],{"type":150,"tag":169,"props":44020,"children":44022},{"className":44021},[177],[44023,44071],{"type":150,"tag":169,"props":44024,"children":44026},{"className":44025},[182],[44027],{"type":150,"tag":185,"props":44028,"children":44029},{"xmlns":187,"display":188},[44030],{"type":150,"tag":191,"props":44031,"children":44032},{},[44033,44066],{"type":150,"tag":195,"props":44034,"children":44035},{},[44036,44054,44058,44062],{"type":150,"tag":1952,"props":44037,"children":44038},{},[44039,44050],{"type":150,"tag":195,"props":44040,"children":44041},{},[44042,44046],{"type":150,"tag":411,"props":44043,"children":44044},{},[44045],{"type":155,"value":9389},{"type":150,"tag":217,"props":44047,"children":44048},{},[44049],{"type":155,"value":29467},{"type":150,"tag":411,"props":44051,"children":44052},{},[44053],{"type":155,"value":22571},{"type":150,"tag":417,"props":44055,"children":44056},{},[44057],{"type":155,"value":1045},{"type":150,"tag":411,"props":44059,"children":44060},{},[44061],{"type":155,"value":433},{"type":150,"tag":217,"props":44063,"children":44064},{},[44065],{"type":155,"value":29467},{"type":150,"tag":223,"props":44067,"children":44068},{"encoding":225},[44069],{"type":155,"value":44070},"\\frac{8\\ \\text{GB}}{4} = 2\\ \\text{GB}",{"type":150,"tag":169,"props":44072,"children":44074},{"className":44073,"ariaHidden":214},[232],[44075,44197],{"type":150,"tag":169,"props":44076,"children":44078},{"className":44077},[237],[44079,44083,44184,44188,44193],{"type":150,"tag":169,"props":44080,"children":44082},{"className":44081,"style":31864},[242],[],{"type":150,"tag":169,"props":44084,"children":44086},{"className":44085},[248],[44087,44091,44180],{"type":150,"tag":169,"props":44088,"children":44090},{"className":44089},[1004,2660],[],{"type":150,"tag":169,"props":44092,"children":44094},{"className":44093},[1952],[44095],{"type":150,"tag":169,"props":44096,"children":44098},{"className":44097},[253,254],[44099,44169],{"type":150,"tag":169,"props":44100,"children":44102},{"className":44101},[259],[44103,44164],{"type":150,"tag":169,"props":44104,"children":44106},{"className":44105,"style":23685},[264],[44107,44123,44134],{"type":150,"tag":169,"props":44108,"children":44109},{"style":8422},[44110,44114],{"type":150,"tag":169,"props":44111,"children":44113},{"className":44112,"style":8427},[274],[],{"type":150,"tag":169,"props":44115,"children":44117},{"className":44116},[248],[44118],{"type":150,"tag":169,"props":44119,"children":44121},{"className":44120},[248],[44122],{"type":155,"value":22571},{"type":150,"tag":169,"props":44124,"children":44125},{"style":8440},[44126,44130],{"type":150,"tag":169,"props":44127,"children":44129},{"className":44128,"style":8427},[274],[],{"type":150,"tag":169,"props":44131,"children":44133},{"className":44132,"style":2982},[2981],[],{"type":150,"tag":169,"props":44135,"children":44136},{"style":8452},[44137,44141],{"type":150,"tag":169,"props":44138,"children":44140},{"className":44139,"style":8427},[274],[],{"type":150,"tag":169,"props":44142,"children":44144},{"className":44143},[248],[44145,44150,44155],{"type":150,"tag":169,"props":44146,"children":44148},{"className":44147},[248],[44149],{"type":155,"value":9389},{"type":150,"tag":169,"props":44151,"children":44153},{"className":44152},[558],[44154],{"type":155,"value":8763},{"type":150,"tag":169,"props":44156,"children":44158},{"className":44157},[248,155],[44159],{"type":150,"tag":169,"props":44160,"children":44162},{"className":44161},[248],[44163],{"type":155,"value":29613},{"type":150,"tag":169,"props":44165,"children":44167},{"className":44166},[355],[44168],{"type":155,"value":358},{"type":150,"tag":169,"props":44170,"children":44172},{"className":44171},[259],[44173],{"type":150,"tag":169,"props":44174,"children":44176},{"className":44175,"style":8484},[264],[44177],{"type":150,"tag":169,"props":44178,"children":44179},{},[],{"type":150,"tag":169,"props":44181,"children":44183},{"className":44182},[1015,2660],[],{"type":150,"tag":169,"props":44185,"children":44187},{"className":44186,"style":839},[558],[],{"type":150,"tag":169,"props":44189,"children":44191},{"className":44190},[844],[44192],{"type":155,"value":1045},{"type":150,"tag":169,"props":44194,"children":44196},{"className":44195,"style":839},[558],[],{"type":150,"tag":169,"props":44198,"children":44200},{"className":44199},[237],[44201,44205,44210,44215],{"type":150,"tag":169,"props":44202,"children":44204},{"className":44203,"style":5854},[242],[],{"type":150,"tag":169,"props":44206,"children":44208},{"className":44207},[248],[44209],{"type":155,"value":433},{"type":150,"tag":169,"props":44211,"children":44213},{"className":44212},[558],[44214],{"type":155,"value":8763},{"type":150,"tag":169,"props":44216,"children":44218},{"className":44217},[248,155],[44219],{"type":150,"tag":169,"props":44220,"children":44222},{"className":44221},[248],[44223],{"type":155,"value":29613},{"type":150,"tag":151,"props":44225,"children":44226},{},[44227],{"type":155,"value":44228},"KV 是同一个道理。TP 把 KV head 也分到四张卡上，每卡只存自己那部分，也就只读自己那部分：",{"type":150,"tag":169,"props":44230,"children":44232},{"className":44231},[172],[44233],{"type":150,"tag":169,"props":44234,"children":44236},{"className":44235},[177],[44237,44301],{"type":150,"tag":169,"props":44238,"children":44240},{"className":44239},[182],[44241],{"type":150,"tag":185,"props":44242,"children":44243},{"xmlns":187,"display":188},[44244],{"type":150,"tag":191,"props":44245,"children":44246},{},[44247,44296],{"type":150,"tag":195,"props":44248,"children":44249},{},[44250,44283,44287,44292],{"type":150,"tag":1952,"props":44251,"children":44252},{},[44253,44279],{"type":150,"tag":195,"props":44254,"children":44255},{},[44256,44260,44264,44268],{"type":150,"tag":411,"props":44257,"children":44258},{},[44259],{"type":155,"value":39544},{"type":150,"tag":217,"props":44261,"children":44262},{},[44263],{"type":155,"value":40732},{"type":150,"tag":417,"props":44265,"children":44266},{},[44267],{"type":155,"value":5554},{"type":150,"tag":746,"props":44269,"children":44270},{},[44271,44275],{"type":150,"tag":411,"props":44272,"children":44273},{},[44274],{"type":155,"value":25792},{"type":150,"tag":411,"props":44276,"children":44277},{},[44278],{"type":155,"value":39560},{"type":150,"tag":411,"props":44280,"children":44281},{},[44282],{"type":155,"value":22571},{"type":150,"tag":417,"props":44284,"children":44285},{},[44286],{"type":155,"value":1045},{"type":150,"tag":411,"props":44288,"children":44289},{},[44290],{"type":155,"value":44291},"0.23",{"type":150,"tag":217,"props":44293,"children":44294},{},[44295],{"type":155,"value":29467},{"type":150,"tag":223,"props":44297,"children":44298},{"encoding":225},[44299],{"type":155,"value":44300},"\\frac{890\\ \\text{B} \\times 10^6}{4} = 0.23\\ \\text{GB}",{"type":150,"tag":169,"props":44302,"children":44304},{"className":44303,"ariaHidden":214},[232],[44305,44487],{"type":150,"tag":169,"props":44306,"children":44308},{"className":44307},[237],[44309,44314,44474,44478,44483],{"type":150,"tag":169,"props":44310,"children":44313},{"className":44311,"style":44312},[242],"height:2.1771em;vertical-align:-0.686em;",[],{"type":150,"tag":169,"props":44315,"children":44317},{"className":44316},[248],[44318,44322,44470],{"type":150,"tag":169,"props":44319,"children":44321},{"className":44320},[1004,2660],[],{"type":150,"tag":169,"props":44323,"children":44325},{"className":44324},[1952],[44326],{"type":150,"tag":169,"props":44327,"children":44329},{"className":44328},[253,254],[44330,44459],{"type":150,"tag":169,"props":44331,"children":44333},{"className":44332},[259],[44334,44454],{"type":150,"tag":169,"props":44335,"children":44337},{"className":44336,"style":25954},[264],[44338,44354,44365],{"type":150,"tag":169,"props":44339,"children":44340},{"style":8422},[44341,44345],{"type":150,"tag":169,"props":44342,"children":44344},{"className":44343,"style":8427},[274],[],{"type":150,"tag":169,"props":44346,"children":44348},{"className":44347},[248],[44349],{"type":150,"tag":169,"props":44350,"children":44352},{"className":44351},[248],[44353],{"type":155,"value":22571},{"type":150,"tag":169,"props":44355,"children":44356},{"style":8440},[44357,44361],{"type":150,"tag":169,"props":44358,"children":44360},{"className":44359,"style":8427},[274],[],{"type":150,"tag":169,"props":44362,"children":44364},{"className":44363,"style":2982},[2981],[],{"type":150,"tag":169,"props":44366,"children":44367},{"style":8452},[44368,44372],{"type":150,"tag":169,"props":44369,"children":44371},{"className":44370,"style":8427},[274],[],{"type":150,"tag":169,"props":44373,"children":44375},{"className":44374},[248],[44376,44381,44386,44395,44399,44404,44408,44413],{"type":150,"tag":169,"props":44377,"children":44379},{"className":44378},[248],[44380],{"type":155,"value":39544},{"type":150,"tag":169,"props":44382,"children":44384},{"className":44383},[558],[44385],{"type":155,"value":8763},{"type":150,"tag":169,"props":44387,"children":44389},{"className":44388},[248,155],[44390],{"type":150,"tag":169,"props":44391,"children":44393},{"className":44392},[248],[44394],{"type":155,"value":24918},{"type":150,"tag":169,"props":44396,"children":44398},{"className":44397,"style":5167},[558],[],{"type":150,"tag":169,"props":44400,"children":44402},{"className":44401},[5172],[44403],{"type":155,"value":5554},{"type":150,"tag":169,"props":44405,"children":44407},{"className":44406,"style":5167},[558],[],{"type":150,"tag":169,"props":44409,"children":44411},{"className":44410},[248],[44412],{"type":155,"value":415},{"type":150,"tag":169,"props":44414,"children":44416},{"className":44415},[248],[44417,44422],{"type":150,"tag":169,"props":44418,"children":44420},{"className":44419},[248],[44421],{"type":155,"value":206},{"type":150,"tag":169,"props":44423,"children":44425},{"className":44424},[495],[44426],{"type":150,"tag":169,"props":44427,"children":44429},{"className":44428},[253],[44430],{"type":150,"tag":169,"props":44431,"children":44433},{"className":44432},[259],[44434],{"type":150,"tag":169,"props":44435,"children":44437},{"className":44436,"style":7005},[264],[44438],{"type":150,"tag":169,"props":44439,"children":44440},{"style":888},[44441,44445],{"type":150,"tag":169,"props":44442,"children":44444},{"className":44443,"style":517},[274],[],{"type":150,"tag":169,"props":44446,"children":44448},{"className":44447},[522,523,524,525],[44449],{"type":150,"tag":169,"props":44450,"children":44452},{"className":44451},[248,525],[44453],{"type":155,"value":39560},{"type":150,"tag":169,"props":44455,"children":44457},{"className":44456},[355],[44458],{"type":155,"value":358},{"type":150,"tag":169,"props":44460,"children":44462},{"className":44461},[259],[44463],{"type":150,"tag":169,"props":44464,"children":44466},{"className":44465,"style":8484},[264],[44467],{"type":150,"tag":169,"props":44468,"children":44469},{},[],{"type":150,"tag":169,"props":44471,"children":44473},{"className":44472},[1015,2660],[],{"type":150,"tag":169,"props":44475,"children":44477},{"className":44476,"style":839},[558],[],{"type":150,"tag":169,"props":44479,"children":44481},{"className":44480},[844],[44482],{"type":155,"value":1045},{"type":150,"tag":169,"props":44484,"children":44486},{"className":44485,"style":839},[558],[],{"type":150,"tag":169,"props":44488,"children":44490},{"className":44489},[237],[44491,44495,44500,44505],{"type":150,"tag":169,"props":44492,"children":44494},{"className":44493,"style":5854},[242],[],{"type":150,"tag":169,"props":44496,"children":44498},{"className":44497},[248],[44499],{"type":155,"value":44291},{"type":150,"tag":169,"props":44501,"children":44503},{"className":44502},[558],[44504],{"type":155,"value":8763},{"type":150,"tag":169,"props":44506,"children":44508},{"className":44507},[248,155],[44509],{"type":150,"tag":169,"props":44510,"children":44512},{"className":44511},[248],[44513],{"type":155,"value":29613},{"type":150,"tag":151,"props":44515,"children":44516},{},[44517,44522],{"type":150,"tag":158,"props":44518,"children":44519},{},[44520],{"type":155,"value":44521},"两项加起来，KV 只占 10%。",{"type":155,"value":44523}," 和前面 batch 1 的结论一致，TP 不改变这个比例。",{"type":150,"tag":151,"props":44525,"children":44526},{},[44527],{"type":155,"value":44528},"四张卡合起来，节点级 KV 容量是 444 GB，按 890 字节 \u002F token 算就是 4.99 亿 token。单序列的理论上限也从 896 提到约 3,600 token\u002Fs。",{"type":150,"tag":151,"props":44530,"children":44531},{},[44532],{"type":155,"value":44533},"这就是 ds4.1f-FP4 在 B200 上的 roofline。",{"type":150,"tag":151,"props":44535,"children":44536},{},[44537,44539,44544],{"type":155,"value":44538},"拐点 1125 FLOP\u002Fbyte 换一种读法：",{"type":150,"tag":158,"props":44540,"children":44541},{},[44542],{"type":155,"value":44543},"搬运 1 个字节的时间，够做 1125 次浮点运算",{"type":155,"value":44545},"。于是一个机制只要能用不到 1125 次运算省下 1 个字节的搬运，就净赚。以这个比值衡量，ds4.1f 里的几个设计都在做同一笔交易：",{"type":150,"tag":23833,"props":44547,"children":44548},{},[44549,44671,44681],{"type":150,"tag":23837,"props":44550,"children":44551},{},[44552,44557,44559,44669],{"type":150,"tag":158,"props":44553,"children":44554},{},[44555],{"type":155,"value":44556},"SWA Bounded Replay",{"type":155,"value":44558},"：请求恢复时不重放整条序列，只重算最近 ",{"type":150,"tag":169,"props":44560,"children":44562},{"className":44561},[177],[44563,44592],{"type":150,"tag":169,"props":44564,"children":44566},{"className":44565},[182],[44567],{"type":150,"tag":185,"props":44568,"children":44569},{"xmlns":187},[44570],{"type":150,"tag":191,"props":44571,"children":44572},{},[44573,44587],{"type":150,"tag":195,"props":44574,"children":44575},{},[44576],{"type":150,"tag":401,"props":44577,"children":44578},{},[44579,44583],{"type":150,"tag":405,"props":44580,"children":44581},{},[44582],{"type":155,"value":1817},{"type":150,"tag":217,"props":44584,"children":44585},{},[44586],{"type":155,"value":38603},{"type":150,"tag":223,"props":44588,"children":44589},{"encoding":225},[44590],{"type":155,"value":44591},"n_{\\text{win}}",{"type":150,"tag":169,"props":44593,"children":44595},{"className":44594,"ariaHidden":214},[232],[44596],{"type":150,"tag":169,"props":44597,"children":44599},{"className":44598},[237],[44600,44604],{"type":150,"tag":169,"props":44601,"children":44603},{"className":44602,"style":1343},[242],[],{"type":150,"tag":169,"props":44605,"children":44607},{"className":44606},[248],[44608,44613],{"type":150,"tag":169,"props":44609,"children":44611},{"className":44610},[248,489],[44612],{"type":155,"value":1817},{"type":150,"tag":169,"props":44614,"children":44616},{"className":44615},[495],[44617],{"type":150,"tag":169,"props":44618,"children":44620},{"className":44619},[253,254],[44621,44658],{"type":150,"tag":169,"props":44622,"children":44624},{"className":44623},[259],[44625,44653],{"type":150,"tag":169,"props":44626,"children":44628},{"className":44627,"style":6478},[264],[44629],{"type":150,"tag":169,"props":44630,"children":44631},{"style":512},[44632,44636],{"type":150,"tag":169,"props":44633,"children":44635},{"className":44634,"style":517},[274],[],{"type":150,"tag":169,"props":44637,"children":44639},{"className":44638},[522,523,524,525],[44640],{"type":150,"tag":169,"props":44641,"children":44643},{"className":44642},[248,525],[44644],{"type":150,"tag":169,"props":44645,"children":44647},{"className":44646},[248,155,525],[44648],{"type":150,"tag":169,"props":44649,"children":44651},{"className":44650},[248,525],[44652],{"type":155,"value":38603},{"type":150,"tag":169,"props":44654,"children":44656},{"className":44655},[355],[44657],{"type":155,"value":358},{"type":150,"tag":169,"props":44659,"children":44661},{"className":44660},[259],[44662],{"type":150,"tag":169,"props":44663,"children":44665},{"className":44664,"style":544},[264],[44666],{"type":150,"tag":169,"props":44667,"children":44668},{},[],{"type":155,"value":44670}," 个 token 来重建局部状态。用一段计算，换掉局部窗口 KV 的持久化。",{"type":150,"tag":23837,"props":44672,"children":44673},{},[44674,44679],{"type":150,"tag":158,"props":44675,"children":44676},{},[44677],{"type":155,"value":44678},"FP4 KV",{"type":155,"value":44680},"：存储和带宽都降到四分之一，代价是读进来要反量化。用一点计算，换掉四分之三的字节。",{"type":150,"tag":23837,"props":44682,"children":44683},{},[44684,44689],{"type":150,"tag":158,"props":44685,"children":44686},{},[44687],{"type":155,"value":44688},"Hierarchical Indexer",{"type":155,"value":44690},"：先用一层打分粗筛出候选池，后续层只在这个池子里挑。用一层索引，换掉对全部候选的扫描。",{"type":150,"tag":151,"props":44692,"children":44693},{},[44694],{"type":155,"value":44695},"共同点是把成本从 memory 一侧挪到 compute 一侧。",{"type":150,"tag":151,"props":44697,"children":44698},{},[44699,44701],{"type":155,"value":44700},"前面几代的动作一直是压缩存储。现在存储已经压到接近下限，规则反过来：",{"type":150,"tag":158,"props":44702,"children":44703},{},[44704],{"type":155,"value":44705},"用富余的算力，去重建那些不值得搬过来的字节。",{"type":150,"tag":151,"props":44707,"children":44708},{},[44709],{"type":155,"value":44710},"这些省下来的资源最终可以被重新花到：更多 tokens、更好的数据过滤、更长的 RL rollout、更多 environment interaction、更多 rejection sampling、更多 verifier compute、更大的 batch、更复杂的 agent tasks。到这里，可以回到最初的问题了。ds4.1f 省掉的，是账单上几乎每一项。",{"type":150,"tag":372,"props":44712,"children":44714},{"id":44713},"capability-与-efficiency",[44715],{"type":155,"value":44716},"Capability 与 Efficiency",{"type":150,"tag":151,"props":44718,"children":44719},{},[44720],{"type":155,"value":44721},"业界基本有共识：能力里很大的一部分来自后训练，而不是架构。三个例子。",{"type":150,"tag":151,"props":44723,"children":44724},{},[44725,44727,44734],{"type":155,"value":44726},"GLM-5.3 的发布说明只有一句话，",{"type":150,"tag":9850,"props":44728,"children":44731},{"href":44729,"rel":44730},"https:\u002F\u002Fz.ai\u002Fblog\u002Fglm-5.3",[9854],[44732],{"type":155,"value":44733},"Scaling post-training is all we did",{"type":155,"value":44735},"。基座和 GLM-5.2 一样，没有新增一个预训练 token，Terminal-Bench 3.0 从 4.6 涨到 28.3，六倍。",{"type":150,"tag":151,"props":44737,"children":44738},{},[44739,44741,44748],{"type":155,"value":44740},"Cognition 的 ",{"type":150,"tag":9850,"props":44742,"children":44745},{"href":44743,"rel":44744},"https:\u002F\u002Fcognition.com\u002Fblog\u002Fswe-2",[9854],[44746],{"type":155,"value":44747},"SWE-2",{"type":155,"value":44749}," 更能说明分工。它是一个美国实验室从 Kimi K3 的开源权重上后训练出来的，而 Kimi K3 是 Moonshot 放出来的 2.8T 参数模型。Cognition 报的成绩是编码能力离前沿只差一点，成本低 64%；首次真正动手改代码的中位步数从 48 降到 18。",{"type":150,"tag":151,"props":44751,"children":44752},{},[44753],{"type":155,"value":44754},"开源权重到了这个水平，做一款有竞争力的编码产品，贵的环节已经不是预训练，而是后训练和 harness。",{"type":150,"tag":151,"props":44756,"children":44757},{},[44758],{"type":155,"value":44759},"第三个例子是 DeepSeek V4-Flash-0731。它的模型结构和尺寸与 Preview 保持一致，只重新做了后训练，但 Agent benchmark 大幅提升：",{"type":150,"tag":169,"props":44761,"children":44763},{"className":44762},[172],[44764],{"type":150,"tag":169,"props":44765,"children":44767},{"className":44766},[177],[44768,44805],{"type":150,"tag":169,"props":44769,"children":44771},{"className":44770},[182],[44772],{"type":150,"tag":185,"props":44773,"children":44774},{"xmlns":187,"display":188},[44775],{"type":150,"tag":191,"props":44776,"children":44777},{},[44778,44800],{"type":150,"tag":195,"props":44779,"children":44780},{},[44781,44786,44791,44795],{"type":150,"tag":217,"props":44782,"children":44783},{},[44784],{"type":155,"value":44785},"DeepSWE: ",{"type":150,"tag":411,"props":44787,"children":44788},{},[44789],{"type":155,"value":44790},"7.3",{"type":150,"tag":417,"props":44792,"children":44793},{},[44794],{"type":155,"value":12954},{"type":150,"tag":411,"props":44796,"children":44797},{},[44798],{"type":155,"value":44799},"54.4",{"type":150,"tag":223,"props":44801,"children":44802},{"encoding":225},[44803],{"type":155,"value":44804},"\\text{DeepSWE: } 7.3 \\rightarrow 54.4",{"type":150,"tag":169,"props":44806,"children":44808},{"className":44807,"ariaHidden":214},[232],[44809,44844],{"type":150,"tag":169,"props":44810,"children":44812},{"className":44811},[237],[44813,44817,44826,44831,44835,44840],{"type":150,"tag":169,"props":44814,"children":44816},{"className":44815,"style":11723},[242],[],{"type":150,"tag":169,"props":44818,"children":44820},{"className":44819},[248,155],[44821],{"type":150,"tag":169,"props":44822,"children":44824},{"className":44823},[248],[44825],{"type":155,"value":44785},{"type":150,"tag":169,"props":44827,"children":44829},{"className":44828},[248],[44830],{"type":155,"value":44790},{"type":150,"tag":169,"props":44832,"children":44834},{"className":44833,"style":839},[558],[],{"type":150,"tag":169,"props":44836,"children":44838},{"className":44837},[844],[44839],{"type":155,"value":12954},{"type":150,"tag":169,"props":44841,"children":44843},{"className":44842,"style":839},[558],[],{"type":150,"tag":169,"props":44845,"children":44847},{"className":44846},[237],[44848,44852],{"type":150,"tag":169,"props":44849,"children":44851},{"className":44850,"style":5186},[242],[],{"type":150,"tag":169,"props":44853,"children":44855},{"className":44854},[248],[44856],{"type":155,"value":44799},{"type":150,"tag":169,"props":44858,"children":44860},{"className":44859},[172],[44861],{"type":150,"tag":169,"props":44862,"children":44864},{"className":44863},[177],[44865,44902],{"type":150,"tag":169,"props":44866,"children":44868},{"className":44867},[182],[44869],{"type":150,"tag":185,"props":44870,"children":44871},{"xmlns":187,"display":188},[44872],{"type":150,"tag":191,"props":44873,"children":44874},{},[44875,44897],{"type":150,"tag":195,"props":44876,"children":44877},{},[44878,44883,44888,44892],{"type":150,"tag":217,"props":44879,"children":44880},{},[44881],{"type":155,"value":44882},"CyberGym: ",{"type":150,"tag":411,"props":44884,"children":44885},{},[44886],{"type":155,"value":44887},"38.7",{"type":150,"tag":417,"props":44889,"children":44890},{},[44891],{"type":155,"value":12954},{"type":150,"tag":411,"props":44893,"children":44894},{},[44895],{"type":155,"value":44896},"76.7",{"type":150,"tag":223,"props":44898,"children":44899},{"encoding":225},[44900],{"type":155,"value":44901},"\\text{CyberGym: } 38.7 \\rightarrow 76.7",{"type":150,"tag":169,"props":44903,"children":44905},{"className":44904,"ariaHidden":214},[232],[44906,44941],{"type":150,"tag":169,"props":44907,"children":44909},{"className":44908},[237],[44910,44914,44923,44928,44932,44937],{"type":150,"tag":169,"props":44911,"children":44913},{"className":44912,"style":1121},[242],[],{"type":150,"tag":169,"props":44915,"children":44917},{"className":44916},[248,155],[44918],{"type":150,"tag":169,"props":44919,"children":44921},{"className":44920},[248],[44922],{"type":155,"value":44882},{"type":150,"tag":169,"props":44924,"children":44926},{"className":44925},[248],[44927],{"type":155,"value":44887},{"type":150,"tag":169,"props":44929,"children":44931},{"className":44930,"style":839},[558],[],{"type":150,"tag":169,"props":44933,"children":44935},{"className":44934},[844],[44936],{"type":155,"value":12954},{"type":150,"tag":169,"props":44938,"children":44940},{"className":44939,"style":839},[558],[],{"type":150,"tag":169,"props":44942,"children":44944},{"className":44943},[237],[44945,44949],{"type":150,"tag":169,"props":44946,"children":44948},{"className":44947,"style":5186},[242],[],{"type":150,"tag":169,"props":44950,"children":44952},{"className":44951},[248],[44953],{"type":155,"value":44896},{"type":150,"tag":169,"props":44955,"children":44957},{"className":44956},[172],[44958],{"type":150,"tag":169,"props":44959,"children":44961},{"className":44960},[177],[44962,44999],{"type":150,"tag":169,"props":44963,"children":44965},{"className":44964},[182],[44966],{"type":150,"tag":185,"props":44967,"children":44968},{"xmlns":187,"display":188},[44969],{"type":150,"tag":191,"props":44970,"children":44971},{},[44972,44994],{"type":150,"tag":195,"props":44973,"children":44974},{},[44975,44980,44985,44989],{"type":150,"tag":217,"props":44976,"children":44977},{},[44978],{"type":155,"value":44979},"TerminalBench 2.1: ",{"type":150,"tag":411,"props":44981,"children":44982},{},[44983],{"type":155,"value":44984},"61.8",{"type":150,"tag":417,"props":44986,"children":44987},{},[44988],{"type":155,"value":12954},{"type":150,"tag":411,"props":44990,"children":44991},{},[44992],{"type":155,"value":44993},"82.7",{"type":150,"tag":223,"props":44995,"children":44996},{"encoding":225},[44997],{"type":155,"value":44998},"\\text{TerminalBench 2.1: } 61.8 \\rightarrow 82.7",{"type":150,"tag":169,"props":45000,"children":45002},{"className":45001,"ariaHidden":214},[232],[45003,45038],{"type":150,"tag":169,"props":45004,"children":45006},{"className":45005},[237],[45007,45011,45020,45025,45029,45034],{"type":150,"tag":169,"props":45008,"children":45010},{"className":45009,"style":942},[242],[],{"type":150,"tag":169,"props":45012,"children":45014},{"className":45013},[248,155],[45015],{"type":150,"tag":169,"props":45016,"children":45018},{"className":45017},[248],[45019],{"type":155,"value":44979},{"type":150,"tag":169,"props":45021,"children":45023},{"className":45022},[248],[45024],{"type":155,"value":44984},{"type":150,"tag":169,"props":45026,"children":45028},{"className":45027,"style":839},[558],[],{"type":150,"tag":169,"props":45030,"children":45032},{"className":45031},[844],[45033],{"type":155,"value":12954},{"type":150,"tag":169,"props":45035,"children":45037},{"className":45036,"style":839},[558],[],{"type":150,"tag":169,"props":45039,"children":45041},{"className":45040},[237],[45042,45046],{"type":150,"tag":169,"props":45043,"children":45045},{"className":45044,"style":5186},[242],[],{"type":150,"tag":169,"props":45047,"children":45049},{"className":45048},[248],[45050],{"type":155,"value":44993},{"type":150,"tag":151,"props":45052,"children":45053},{},[45054,45056,45061],{"type":155,"value":45055},"模型 backbone 没换。Attention 没换。参数量没换。官方更新日志直接写明结构不变、仅重新后训练。这个例子比任何口号都说明问题：",{"type":150,"tag":158,"props":45057,"children":45058},{},[45059],{"type":155,"value":45060},"同一个 backbone 的 post-training 差异，完全可能比两代 architecture 的差异更大。",{"type":155,"value":45062}," 所以把\"模型进步\"拆成四个层次更合适：",{"type":150,"tag":22368,"props":45064,"children":45065},{},[45066,45087],{"type":150,"tag":22372,"props":45067,"children":45068},{},[45069],{"type":150,"tag":22376,"props":45070,"children":45071},{},[45072,45077,45082],{"type":150,"tag":22380,"props":45073,"children":45074},{},[45075],{"type":155,"value":45076},"层次",{"type":150,"tag":22380,"props":45078,"children":45079},{},[45080],{"type":155,"value":45081},"解决什么",{"type":150,"tag":22380,"props":45083,"children":45084},{},[45085],{"type":155,"value":45086},"对能力的作用",{"type":150,"tag":22394,"props":45088,"children":45089},{},[45090,45108,45126,45144],{"type":150,"tag":22376,"props":45091,"children":45092},{},[45093,45098,45103],{"type":150,"tag":22401,"props":45094,"children":45095},{},[45096],{"type":155,"value":45097},"Architecture",{"type":150,"tag":22401,"props":45099,"children":45100},{},[45101],{"type":155,"value":45102},"每 FLOP \u002F 每 GB 能做多少事",{"type":150,"tag":22401,"props":45104,"children":45105},{},[45106],{"type":155,"value":45107},"提高上限和效率",{"type":150,"tag":22376,"props":45109,"children":45110},{},[45111,45116,45121],{"type":150,"tag":22401,"props":45112,"children":45113},{},[45114],{"type":155,"value":45115},"Data",{"type":150,"tag":22401,"props":45117,"children":45118},{},[45119],{"type":155,"value":45120},"模型到底看到了什么分布",{"type":150,"tag":22401,"props":45122,"children":45123},{},[45124],{"type":155,"value":45125},"决定知识、能力覆盖",{"type":150,"tag":22376,"props":45127,"children":45128},{},[45129,45134,45139],{"type":150,"tag":22401,"props":45130,"children":45131},{},[45132],{"type":155,"value":45133},"Post-training \u002F RL",{"type":150,"tag":22401,"props":45135,"children":45136},{},[45137],{"type":155,"value":45138},"学会怎么推理、用工具、完成任务",{"type":150,"tag":22401,"props":45140,"children":45141},{},[45142],{"type":155,"value":45143},"直接决定最终行为能力",{"type":150,"tag":22376,"props":45145,"children":45146},{},[45147,45152,45157],{"type":150,"tag":22401,"props":45148,"children":45149},{},[45150],{"type":155,"value":45151},"Infra",{"type":150,"tag":22401,"props":45153,"children":45154},{},[45155],{"type":155,"value":45156},"上面三件事是否真的按设计运行",{"type":150,"tag":22401,"props":45158,"children":45159},{},[45160],{"type":155,"value":45161},"防止几个月训练被 bug 偷走",{"type":150,"tag":151,"props":45163,"children":45164},{},[45165],{"type":155,"value":45166},"Infra 这一层最容易被低估，也最残酷。因为 RL 不是：",{"type":150,"tag":169,"props":45168,"children":45170},{"className":45169},[172],[45171],{"type":150,"tag":169,"props":45172,"children":45174},{"className":45173},[177],[45175,45207],{"type":150,"tag":169,"props":45176,"children":45178},{"className":45177},[182],[45179],{"type":150,"tag":185,"props":45180,"children":45181},{"xmlns":187,"display":188},[45182],{"type":150,"tag":191,"props":45183,"children":45184},{},[45185,45202],{"type":150,"tag":195,"props":45186,"children":45187},{},[45188,45193,45197],{"type":150,"tag":217,"props":45189,"children":45190},{},[45191],{"type":155,"value":45192},"写个 PPO\u002FGRPO 公式",{"type":150,"tag":417,"props":45194,"children":45195},{},[45196],{"type":155,"value":12954},{"type":150,"tag":217,"props":45198,"children":45199},{},[45200],{"type":155,"value":45201},"模型自动变聪明",{"type":150,"tag":223,"props":45203,"children":45204},{"encoding":225},[45205],{"type":155,"value":45206},"\\text{写个 PPO\u002FGRPO 公式} \\rightarrow \\text{模型自动变聪明}",{"type":150,"tag":169,"props":45208,"children":45210},{"className":45209,"ariaHidden":214},[232],[45211,45254],{"type":150,"tag":169,"props":45212,"children":45214},{"className":45213},[237],[45215,45219,45241,45245,45250],{"type":150,"tag":169,"props":45216,"children":45218},{"className":45217,"style":999},[242],[],{"type":150,"tag":169,"props":45220,"children":45222},{"className":45221},[248,155],[45223,45229,45235],{"type":150,"tag":169,"props":45224,"children":45226},{"className":45225},[248,297],[45227],{"type":155,"value":45228},"写个",{"type":150,"tag":169,"props":45230,"children":45232},{"className":45231},[248],[45233],{"type":155,"value":45234}," PPO\u002FGRPO ",{"type":150,"tag":169,"props":45236,"children":45238},{"className":45237},[248,297],[45239],{"type":155,"value":45240},"公式",{"type":150,"tag":169,"props":45242,"children":45244},{"className":45243,"style":839},[558],[],{"type":150,"tag":169,"props":45246,"children":45248},{"className":45247},[844],[45249],{"type":155,"value":12954},{"type":150,"tag":169,"props":45251,"children":45253},{"className":45252,"style":839},[558],[],{"type":150,"tag":169,"props":45255,"children":45257},{"className":45256},[237],[45258,45262],{"type":150,"tag":169,"props":45259,"children":45261},{"className":45260,"style":5854},[242],[],{"type":150,"tag":169,"props":45263,"children":45265},{"className":45264},[248,155],[45266],{"type":150,"tag":169,"props":45267,"children":45269},{"className":45268},[248,297],[45270],{"type":155,"value":45201},{"type":150,"tag":151,"props":45272,"children":45273},{},[45274],{"type":155,"value":45275},"而是一条很长的系统：",{"type":150,"tag":169,"props":45277,"children":45279},{"className":45278},[172],[45280],{"type":150,"tag":169,"props":45281,"children":45283},{"className":45282},[177],[45284,45370],{"type":150,"tag":169,"props":45285,"children":45287},{"className":45286},[182],[45288],{"type":150,"tag":185,"props":45289,"children":45290},{"xmlns":187,"display":188},[45291],{"type":150,"tag":191,"props":45292,"children":45293},{},[45294,45365],{"type":150,"tag":195,"props":45295,"children":45296},{},[45297,45302,45306,45311,45315,45320,45324,45329,45333,45338,45342,45347,45351,45356,45360],{"type":150,"tag":217,"props":45298,"children":45299},{},[45300],{"type":155,"value":45301},"prompt",{"type":150,"tag":417,"props":45303,"children":45304},{},[45305],{"type":155,"value":12954},{"type":150,"tag":217,"props":45307,"children":45308},{},[45309],{"type":155,"value":45310},"rollout",{"type":150,"tag":417,"props":45312,"children":45313},{},[45314],{"type":155,"value":12954},{"type":150,"tag":217,"props":45316,"children":45317},{},[45318],{"type":155,"value":45319},"tool environment",{"type":150,"tag":417,"props":45321,"children":45322},{},[45323],{"type":155,"value":12954},{"type":150,"tag":217,"props":45325,"children":45326},{},[45327],{"type":155,"value":45328},"reward",{"type":150,"tag":417,"props":45330,"children":45331},{},[45332],{"type":155,"value":12954},{"type":150,"tag":217,"props":45334,"children":45335},{},[45336],{"type":155,"value":45337},"sampling",{"type":150,"tag":417,"props":45339,"children":45340},{},[45341],{"type":155,"value":12954},{"type":150,"tag":217,"props":45343,"children":45344},{},[45345],{"type":155,"value":45346},"advantage",{"type":150,"tag":417,"props":45348,"children":45349},{},[45350],{"type":155,"value":12954},{"type":150,"tag":217,"props":45352,"children":45353},{},[45354],{"type":155,"value":45355},"gradient",{"type":150,"tag":417,"props":45357,"children":45358},{},[45359],{"type":155,"value":12954},{"type":150,"tag":217,"props":45361,"children":45362},{},[45363],{"type":155,"value":45364},"model update",{"type":150,"tag":223,"props":45366,"children":45367},{"encoding":225},[45368],{"type":155,"value":45369},"\\text{prompt} \\rightarrow \\text{rollout} \\rightarrow \\text{tool environment} \\rightarrow \\text{reward} \\rightarrow \\text{sampling} \\rightarrow \\text{advantage} \\rightarrow \\text{gradient} \\rightarrow \\text{model update}",{"type":150,"tag":169,"props":45371,"children":45373},{"className":45372,"ariaHidden":214},[232],[45374,45405,45435,45465,45495,45525,45555,45585],{"type":150,"tag":169,"props":45375,"children":45377},{"className":45376},[237],[45378,45383,45392,45396,45401],{"type":150,"tag":169,"props":45379,"children":45382},{"className":45380,"style":45381},[242],"height:0.8095em;vertical-align:-0.1944em;",[],{"type":150,"tag":169,"props":45384,"children":45386},{"className":45385},[248,155],[45387],{"type":150,"tag":169,"props":45388,"children":45390},{"className":45389},[248],[45391],{"type":155,"value":45301},{"type":150,"tag":169,"props":45393,"children":45395},{"className":45394,"style":839},[558],[],{"type":150,"tag":169,"props":45397,"children":45399},{"className":45398},[844],[45400],{"type":155,"value":12954},{"type":150,"tag":169,"props":45402,"children":45404},{"className":45403,"style":839},[558],[],{"type":150,"tag":169,"props":45406,"children":45408},{"className":45407},[237],[45409,45413,45422,45426,45431],{"type":150,"tag":169,"props":45410,"children":45412},{"className":45411,"style":942},[242],[],{"type":150,"tag":169,"props":45414,"children":45416},{"className":45415},[248,155],[45417],{"type":150,"tag":169,"props":45418,"children":45420},{"className":45419},[248],[45421],{"type":155,"value":45310},{"type":150,"tag":169,"props":45423,"children":45425},{"className":45424,"style":839},[558],[],{"type":150,"tag":169,"props":45427,"children":45429},{"className":45428},[844],[45430],{"type":155,"value":12954},{"type":150,"tag":169,"props":45432,"children":45434},{"className":45433,"style":839},[558],[],{"type":150,"tag":169,"props":45436,"children":45438},{"className":45437},[237],[45439,45443,45452,45456,45461],{"type":150,"tag":169,"props":45440,"children":45442},{"className":45441,"style":942},[242],[],{"type":150,"tag":169,"props":45444,"children":45446},{"className":45445},[248,155],[45447],{"type":150,"tag":169,"props":45448,"children":45450},{"className":45449},[248],[45451],{"type":155,"value":45319},{"type":150,"tag":169,"props":45453,"children":45455},{"className":45454,"style":839},[558],[],{"type":150,"tag":169,"props":45457,"children":45459},{"className":45458},[844],[45460],{"type":155,"value":12954},{"type":150,"tag":169,"props":45462,"children":45464},{"className":45463,"style":839},[558],[],{"type":150,"tag":169,"props":45466,"children":45468},{"className":45467},[237],[45469,45473,45482,45486,45491],{"type":150,"tag":169,"props":45470,"children":45472},{"className":45471,"style":942},[242],[],{"type":150,"tag":169,"props":45474,"children":45476},{"className":45475},[248,155],[45477],{"type":150,"tag":169,"props":45478,"children":45480},{"className":45479},[248],[45481],{"type":155,"value":45328},{"type":150,"tag":169,"props":45483,"children":45485},{"className":45484,"style":839},[558],[],{"type":150,"tag":169,"props":45487,"children":45489},{"className":45488},[844],[45490],{"type":155,"value":12954},{"type":150,"tag":169,"props":45492,"children":45494},{"className":45493,"style":839},[558],[],{"type":150,"tag":169,"props":45496,"children":45498},{"className":45497},[237],[45499,45503,45512,45516,45521],{"type":150,"tag":169,"props":45500,"children":45502},{"className":45501,"style":1121},[242],[],{"type":150,"tag":169,"props":45504,"children":45506},{"className":45505},[248,155],[45507],{"type":150,"tag":169,"props":45508,"children":45510},{"className":45509},[248],[45511],{"type":155,"value":45337},{"type":150,"tag":169,"props":45513,"children":45515},{"className":45514,"style":839},[558],[],{"type":150,"tag":169,"props":45517,"children":45519},{"className":45518},[844],[45520],{"type":155,"value":12954},{"type":150,"tag":169,"props":45522,"children":45524},{"className":45523,"style":839},[558],[],{"type":150,"tag":169,"props":45526,"children":45528},{"className":45527},[237],[45529,45533,45542,45546,45551],{"type":150,"tag":169,"props":45530,"children":45532},{"className":45531,"style":1121},[242],[],{"type":150,"tag":169,"props":45534,"children":45536},{"className":45535},[248,155],[45537],{"type":150,"tag":169,"props":45538,"children":45540},{"className":45539},[248],[45541],{"type":155,"value":45346},{"type":150,"tag":169,"props":45543,"children":45545},{"className":45544,"style":839},[558],[],{"type":150,"tag":169,"props":45547,"children":45549},{"className":45548},[844],[45550],{"type":155,"value":12954},{"type":150,"tag":169,"props":45552,"children":45554},{"className":45553,"style":839},[558],[],{"type":150,"tag":169,"props":45556,"children":45558},{"className":45557},[237],[45559,45563,45572,45576,45581],{"type":150,"tag":169,"props":45560,"children":45562},{"className":45561,"style":1121},[242],[],{"type":150,"tag":169,"props":45564,"children":45566},{"className":45565},[248,155],[45567],{"type":150,"tag":169,"props":45568,"children":45570},{"className":45569},[248],[45571],{"type":155,"value":45355},{"type":150,"tag":169,"props":45573,"children":45575},{"className":45574,"style":839},[558],[],{"type":150,"tag":169,"props":45577,"children":45579},{"className":45578},[844],[45580],{"type":155,"value":12954},{"type":150,"tag":169,"props":45582,"children":45584},{"className":45583,"style":839},[558],[],{"type":150,"tag":169,"props":45586,"children":45588},{"className":45587},[237],[45589,45593],{"type":150,"tag":169,"props":45590,"children":45592},{"className":45591,"style":1121},[242],[],{"type":150,"tag":169,"props":45594,"children":45596},{"className":45595},[248,155],[45597],{"type":150,"tag":169,"props":45598,"children":45600},{"className":45599},[248],[45601],{"type":155,"value":45364},{"type":150,"tag":151,"props":45603,"children":45604},{},[45605],{"type":155,"value":45606},"任何地方一个小 bug，最后都会变成同一句话：\"奇怪，我们这个新的 RL 算法怎么效果一般？\" 而那个 bug 可能是 reward 算错、tool state 泄漏、rollout truncation 错、mask 错、sampling distribution 不一致、tokenizer offset 错、advantage normalization 错、stale policy 太严重，或者 sandbox reset 有问题。实际上不是算法一般，是代码在偷偷犯罪。所以更准确的因果链是：",{"type":150,"tag":169,"props":45608,"children":45610},{"className":45609},[172],[45611],{"type":150,"tag":169,"props":45612,"children":45614},{"className":45613},[177],[45615,45679],{"type":150,"tag":169,"props":45616,"children":45618},{"className":45617},[182],[45619],{"type":150,"tag":185,"props":45620,"children":45621},{"xmlns":187,"display":188},[45622],{"type":150,"tag":191,"props":45623,"children":45624},{},[45625,45674],{"type":150,"tag":195,"props":45626,"children":45627},{},[45628],{"type":150,"tag":199,"props":45629,"children":45630},{"notation":201},[45631],{"type":150,"tag":204,"props":45632,"children":45633},{"scriptlevel":206,"displaystyle":207},[45634],{"type":150,"tag":204,"props":45635,"children":45636},{"scriptlevel":206,"displaystyle":207},[45637],{"type":150,"tag":204,"props":45638,"children":45639},{"scriptlevel":206,"displaystyle":214},[45640],{"type":150,"tag":195,"props":45641,"children":45642},{},[45643,45647,45651,45656,45660,45665,45669],{"type":150,"tag":217,"props":45644,"children":45645},{},[45646],{"type":155,"value":45097},{"type":150,"tag":417,"props":45648,"children":45649},{},[45650],{"type":155,"value":12954},{"type":150,"tag":217,"props":45652,"children":45653},{},[45654],{"type":155,"value":45655},"Cost reduction",{"type":150,"tag":417,"props":45657,"children":45658},{},[45659],{"type":155,"value":12954},{"type":150,"tag":217,"props":45661,"children":45662},{},[45663],{"type":155,"value":45664},"More effective training",{"type":150,"tag":417,"props":45666,"children":45667},{},[45668],{"type":155,"value":12954},{"type":150,"tag":217,"props":45670,"children":45671},{},[45672],{"type":155,"value":45673},"Capability",{"type":150,"tag":223,"props":45675,"children":45676},{"encoding":225},[45677],{"type":155,"value":45678},"\\boxed{\\text{Architecture} \\rightarrow \\text{Cost reduction} \\rightarrow \\text{More effective training} \\rightarrow \\text{Capability}}",{"type":150,"tag":169,"props":45680,"children":45682},{"className":45681,"ariaHidden":214},[232],[45683],{"type":150,"tag":169,"props":45684,"children":45686},{"className":45685},[237],[45687,45691],{"type":150,"tag":169,"props":45688,"children":45690},{"className":45689,"style":30651},[242],[],{"type":150,"tag":169,"props":45692,"children":45694},{"className":45693},[248],[45695],{"type":150,"tag":169,"props":45696,"children":45698},{"className":45697},[253,254],[45699,45817],{"type":150,"tag":169,"props":45700,"children":45702},{"className":45701},[259],[45703,45812],{"type":150,"tag":169,"props":45704,"children":45706},{"className":45705,"style":30668},[264],[45707,45801],{"type":150,"tag":169,"props":45708,"children":45709},{"style":30672},[45710,45714],{"type":150,"tag":169,"props":45711,"children":45713},{"className":45712,"style":30677},[274],[],{"type":150,"tag":169,"props":45715,"children":45717},{"className":45716},[280],[45718],{"type":150,"tag":169,"props":45719,"children":45721},{"className":45720},[248],[45722],{"type":150,"tag":169,"props":45723,"children":45725},{"className":45724},[248],[45726,45735,45739,45744,45748,45757,45761,45766,45770,45779,45783,45788,45792],{"type":150,"tag":169,"props":45727,"children":45729},{"className":45728},[248,155],[45730],{"type":150,"tag":169,"props":45731,"children":45733},{"className":45732},[248],[45734],{"type":155,"value":45097},{"type":150,"tag":169,"props":45736,"children":45738},{"className":45737,"style":839},[558],[],{"type":150,"tag":169,"props":45740,"children":45742},{"className":45741},[844],[45743],{"type":155,"value":12954},{"type":150,"tag":169,"props":45745,"children":45747},{"className":45746,"style":839},[558],[],{"type":150,"tag":169,"props":45749,"children":45751},{"className":45750},[248,155],[45752],{"type":150,"tag":169,"props":45753,"children":45755},{"className":45754},[248],[45756],{"type":155,"value":45655},{"type":150,"tag":169,"props":45758,"children":45760},{"className":45759,"style":839},[558],[],{"type":150,"tag":169,"props":45762,"children":45764},{"className":45763},[844],[45765],{"type":155,"value":12954},{"type":150,"tag":169,"props":45767,"children":45769},{"className":45768,"style":839},[558],[],{"type":150,"tag":169,"props":45771,"children":45773},{"className":45772},[248,155],[45774],{"type":150,"tag":169,"props":45775,"children":45777},{"className":45776},[248],[45778],{"type":155,"value":45664},{"type":150,"tag":169,"props":45780,"children":45782},{"className":45781,"style":839},[558],[],{"type":150,"tag":169,"props":45784,"children":45786},{"className":45785},[844],[45787],{"type":155,"value":12954},{"type":150,"tag":169,"props":45789,"children":45791},{"className":45790,"style":839},[558],[],{"type":150,"tag":169,"props":45793,"children":45795},{"className":45794},[248,155],[45796],{"type":150,"tag":169,"props":45797,"children":45799},{"className":45798},[248],[45800],{"type":155,"value":45673},{"type":150,"tag":169,"props":45802,"children":45803},{"style":30721},[45804,45808],{"type":150,"tag":169,"props":45805,"children":45807},{"className":45806,"style":30677},[274],[],{"type":150,"tag":169,"props":45809,"children":45811},{"className":45810,"style":30730},[348,349],[],{"type":150,"tag":169,"props":45813,"children":45815},{"className":45814},[355],[45816],{"type":155,"value":358},{"type":150,"tag":169,"props":45818,"children":45820},{"className":45819},[259],[45821],{"type":150,"tag":169,"props":45822,"children":45824},{"className":45823,"style":366},[264],[45825],{"type":150,"tag":169,"props":45826,"children":45827},{},[],{"type":150,"tag":151,"props":45829,"children":45830},{},[45831],{"type":155,"value":45832},"而不是：",{"type":150,"tag":169,"props":45834,"children":45836},{"className":45835},[172],[45837],{"type":150,"tag":169,"props":45838,"children":45840},{"className":45839},[177],[45841,45887],{"type":150,"tag":169,"props":45842,"children":45844},{"className":45843},[182],[45845],{"type":150,"tag":185,"props":45846,"children":45847},{"xmlns":187,"display":188},[45848],{"type":150,"tag":191,"props":45849,"children":45850},{},[45851,45882],{"type":150,"tag":195,"props":45852,"children":45853},{},[45854],{"type":150,"tag":199,"props":45855,"children":45856},{"notation":201},[45857],{"type":150,"tag":204,"props":45858,"children":45859},{"scriptlevel":206,"displaystyle":207},[45860],{"type":150,"tag":204,"props":45861,"children":45862},{"scriptlevel":206,"displaystyle":207},[45863],{"type":150,"tag":204,"props":45864,"children":45865},{"scriptlevel":206,"displaystyle":214},[45866],{"type":150,"tag":195,"props":45867,"children":45868},{},[45869,45873,45877],{"type":150,"tag":217,"props":45870,"children":45871},{},[45872],{"type":155,"value":45097},{"type":150,"tag":417,"props":45874,"children":45875},{},[45876],{"type":155,"value":12954},{"type":150,"tag":217,"props":45878,"children":45879},{},[45880],{"type":155,"value":45881},"Magic intelligence",{"type":150,"tag":223,"props":45883,"children":45884},{"encoding":225},[45885],{"type":155,"value":45886},"\\boxed{\\text{Architecture} \\rightarrow \\text{Magic intelligence}}",{"type":150,"tag":169,"props":45888,"children":45890},{"className":45889,"ariaHidden":214},[232],[45891],{"type":150,"tag":169,"props":45892,"children":45894},{"className":45893},[237],[45895,45899],{"type":150,"tag":169,"props":45896,"children":45898},{"className":45897,"style":30651},[242],[],{"type":150,"tag":169,"props":45900,"children":45902},{"className":45901},[248],[45903],{"type":150,"tag":169,"props":45904,"children":45906},{"className":45905},[253,254],[45907,45981],{"type":150,"tag":169,"props":45908,"children":45910},{"className":45909},[259],[45911,45976],{"type":150,"tag":169,"props":45912,"children":45914},{"className":45913,"style":30668},[264],[45915,45965],{"type":150,"tag":169,"props":45916,"children":45917},{"style":30672},[45918,45922],{"type":150,"tag":169,"props":45919,"children":45921},{"className":45920,"style":30677},[274],[],{"type":150,"tag":169,"props":45923,"children":45925},{"className":45924},[280],[45926],{"type":150,"tag":169,"props":45927,"children":45929},{"className":45928},[248],[45930],{"type":150,"tag":169,"props":45931,"children":45933},{"className":45932},[248],[45934,45943,45947,45952,45956],{"type":150,"tag":169,"props":45935,"children":45937},{"className":45936},[248,155],[45938],{"type":150,"tag":169,"props":45939,"children":45941},{"className":45940},[248],[45942],{"type":155,"value":45097},{"type":150,"tag":169,"props":45944,"children":45946},{"className":45945,"style":839},[558],[],{"type":150,"tag":169,"props":45948,"children":45950},{"className":45949},[844],[45951],{"type":155,"value":12954},{"type":150,"tag":169,"props":45953,"children":45955},{"className":45954,"style":839},[558],[],{"type":150,"tag":169,"props":45957,"children":45959},{"className":45958},[248,155],[45960],{"type":150,"tag":169,"props":45961,"children":45963},{"className":45962},[248],[45964],{"type":155,"value":45881},{"type":150,"tag":169,"props":45966,"children":45967},{"style":30721},[45968,45972],{"type":150,"tag":169,"props":45969,"children":45971},{"className":45970,"style":30677},[274],[],{"type":150,"tag":169,"props":45973,"children":45975},{"className":45974,"style":30730},[348,349],[],{"type":150,"tag":169,"props":45977,"children":45979},{"className":45978},[355],[45980],{"type":155,"value":358},{"type":150,"tag":169,"props":45982,"children":45984},{"className":45983},[259],[45985],{"type":150,"tag":169,"props":45986,"children":45988},{"className":45987,"style":366},[264],[45989],{"type":150,"tag":169,"props":45990,"children":45991},{},[],{"type":150,"tag":151,"props":45993,"children":45994},{},[45995,45997,46002],{"type":155,"value":45996},"架构对能力的作用，经常是",{"type":150,"tag":158,"props":45998,"children":45999},{},[46000],{"type":155,"value":46001},"间接的",{"type":155,"value":46003},"。如果只能从这一节记一句话，我会记这个排序：",{"type":150,"tag":169,"props":46005,"children":46007},{"className":46006},[172],[46008],{"type":150,"tag":169,"props":46009,"children":46011},{"className":46010},[177],[46012,46085],{"type":150,"tag":169,"props":46013,"children":46015},{"className":46014},[182],[46016],{"type":150,"tag":185,"props":46017,"children":46018},{"xmlns":187,"display":188},[46019],{"type":150,"tag":191,"props":46020,"children":46021},{},[46022,46080],{"type":150,"tag":195,"props":46023,"children":46024},{},[46025],{"type":150,"tag":199,"props":46026,"children":46027},{"notation":201},[46028],{"type":150,"tag":204,"props":46029,"children":46030},{"scriptlevel":206,"displaystyle":207},[46031],{"type":150,"tag":204,"props":46032,"children":46033},{"scriptlevel":206,"displaystyle":207},[46034],{"type":150,"tag":204,"props":46035,"children":46036},{"scriptlevel":206,"displaystyle":214},[46037],{"type":150,"tag":195,"props":46038,"children":46039},{},[46040,46044,46049,46054,46058,46063,46067,46072,46076],{"type":150,"tag":217,"props":46041,"children":46042},{},[46043],{"type":155,"value":45115},{"type":150,"tag":417,"props":46045,"children":46046},{},[46047],{"type":155,"value":46048},">",{"type":150,"tag":217,"props":46050,"children":46051},{},[46052],{"type":155,"value":46053},"Post-training\u002FRL",{"type":150,"tag":417,"props":46055,"children":46056},{},[46057],{"type":155,"value":46048},{"type":150,"tag":217,"props":46059,"children":46060},{},[46061],{"type":155,"value":46062},"Eval\u002FVerifier",{"type":150,"tag":417,"props":46064,"children":46065},{},[46066],{"type":155,"value":46048},{"type":150,"tag":217,"props":46068,"children":46069},{},[46070],{"type":155,"value":46071},"Training infra",{"type":150,"tag":417,"props":46073,"children":46074},{},[46075],{"type":155,"value":46048},{"type":150,"tag":217,"props":46077,"children":46078},{},[46079],{"type":155,"value":45097},{"type":150,"tag":223,"props":46081,"children":46082},{"encoding":225},[46083],{"type":155,"value":46084},"\\boxed{\\text{Data} > \\text{Post-training\u002FRL} > \\text{Eval\u002FVerifier} > \\text{Training infra} > \\text{Architecture}}",{"type":150,"tag":169,"props":46086,"children":46088},{"className":46087,"ariaHidden":214},[232],[46089],{"type":150,"tag":169,"props":46090,"children":46092},{"className":46091},[237],[46093,46098],{"type":150,"tag":169,"props":46094,"children":46097},{"className":46095,"style":46096},[242],"height:1.68em;vertical-align:-0.59em;",[],{"type":150,"tag":169,"props":46099,"children":46101},{"className":46100},[248],[46102],{"type":150,"tag":169,"props":46103,"children":46105},{"className":46104},[253,254],[46106,46251],{"type":150,"tag":169,"props":46107,"children":46109},{"className":46108},[259],[46110,46246],{"type":150,"tag":169,"props":46111,"children":46114},{"className":46112,"style":46113},[264],"height:1.09em;",[46115,46233],{"type":150,"tag":169,"props":46116,"children":46118},{"style":46117},"top:-3.68em;",[46119,46124],{"type":150,"tag":169,"props":46120,"children":46123},{"className":46121,"style":46122},[274],"height:3.68em;",[],{"type":150,"tag":169,"props":46125,"children":46127},{"className":46126},[280],[46128],{"type":150,"tag":169,"props":46129,"children":46131},{"className":46130},[248],[46132],{"type":150,"tag":169,"props":46133,"children":46135},{"className":46134},[248],[46136,46145,46149,46154,46158,46167,46171,46176,46180,46189,46193,46198,46202,46211,46215,46220,46224],{"type":150,"tag":169,"props":46137,"children":46139},{"className":46138},[248,155],[46140],{"type":150,"tag":169,"props":46141,"children":46143},{"className":46142},[248],[46144],{"type":155,"value":45115},{"type":150,"tag":169,"props":46146,"children":46148},{"className":46147,"style":839},[558],[],{"type":150,"tag":169,"props":46150,"children":46152},{"className":46151},[844],[46153],{"type":155,"value":46048},{"type":150,"tag":169,"props":46155,"children":46157},{"className":46156,"style":839},[558],[],{"type":150,"tag":169,"props":46159,"children":46161},{"className":46160},[248,155],[46162],{"type":150,"tag":169,"props":46163,"children":46165},{"className":46164},[248],[46166],{"type":155,"value":46053},{"type":150,"tag":169,"props":46168,"children":46170},{"className":46169,"style":839},[558],[],{"type":150,"tag":169,"props":46172,"children":46174},{"className":46173},[844],[46175],{"type":155,"value":46048},{"type":150,"tag":169,"props":46177,"children":46179},{"className":46178,"style":839},[558],[],{"type":150,"tag":169,"props":46181,"children":46183},{"className":46182},[248,155],[46184],{"type":150,"tag":169,"props":46185,"children":46187},{"className":46186},[248],[46188],{"type":155,"value":46062},{"type":150,"tag":169,"props":46190,"children":46192},{"className":46191,"style":839},[558],[],{"type":150,"tag":169,"props":46194,"children":46196},{"className":46195},[844],[46197],{"type":155,"value":46048},{"type":150,"tag":169,"props":46199,"children":46201},{"className":46200,"style":839},[558],[],{"type":150,"tag":169,"props":46203,"children":46205},{"className":46204},[248,155],[46206],{"type":150,"tag":169,"props":46207,"children":46209},{"className":46208},[248],[46210],{"type":155,"value":46071},{"type":150,"tag":169,"props":46212,"children":46214},{"className":46213,"style":839},[558],[],{"type":150,"tag":169,"props":46216,"children":46218},{"className":46217},[844],[46219],{"type":155,"value":46048},{"type":150,"tag":169,"props":46221,"children":46223},{"className":46222,"style":839},[558],[],{"type":150,"tag":169,"props":46225,"children":46227},{"className":46226},[248,155],[46228],{"type":150,"tag":169,"props":46229,"children":46231},{"className":46230},[248],[46232],{"type":155,"value":45097},{"type":150,"tag":169,"props":46234,"children":46236},{"style":46235},"top:-3.09em;",[46237,46241],{"type":150,"tag":169,"props":46238,"children":46240},{"className":46239,"style":46122},[274],[],{"type":150,"tag":169,"props":46242,"children":46245},{"className":46243,"style":46244},[348,349],"height:1.68em;border-style:solid;border-width:0.04em;",[],{"type":150,"tag":169,"props":46247,"children":46249},{"className":46248},[355],[46250],{"type":155,"value":358},{"type":150,"tag":169,"props":46252,"children":46254},{"className":46253},[259],[46255],{"type":150,"tag":169,"props":46256,"children":46259},{"className":46257,"style":46258},[264],"height:0.59em;",[46260],{"type":150,"tag":169,"props":46261,"children":46262},{},[],{"type":150,"tag":151,"props":46264,"children":46265},{},[46266],{"type":155,"value":46267},"这不是说 architecture 最不重要。而是如果你问\"这一代模型为什么明显更好用了\"，答案很多时候在前面几项，而不是\"我们把 attention 又改了一个 acronym\"。旁边还有一组更好记的说法，把每一层对应到一个\"前沿\"：",{"type":150,"tag":23833,"props":46269,"children":46270},{},[46271,46283,46295,46307],{"type":150,"tag":23837,"props":46272,"children":46273},{},[46274,46276,46281],{"type":155,"value":46275},"Architecture 决定 ",{"type":150,"tag":158,"props":46277,"children":46278},{},[46279],{"type":155,"value":46280},"efficiency frontier",{"type":155,"value":46282},"：每单位算力能处理多少有效训练信号",{"type":150,"tag":23837,"props":46284,"children":46285},{},[46286,46288,46293],{"type":155,"value":46287},"Data 决定 ",{"type":150,"tag":158,"props":46289,"children":46290},{},[46291],{"type":155,"value":46292},"learning frontier",{"type":155,"value":46294},"：模型有机会学到什么",{"type":150,"tag":23837,"props":46296,"children":46297},{},[46298,46300,46305],{"type":155,"value":46299},"RL \u002F post-training 决定 ",{"type":150,"tag":158,"props":46301,"children":46302},{},[46303],{"type":155,"value":46304},"behavior frontier",{"type":155,"value":46306},"：模型会不会把能力真正用出来",{"type":150,"tag":23837,"props":46308,"children":46309},{},[46310,46312,46317],{"type":155,"value":46311},"Infra 决定",{"type":150,"tag":158,"props":46313,"children":46314},{},[46315],{"type":155,"value":46316},"你是否真的获得前三者",{"type":155,"value":46318},"：实验结果到底是不是你以为的实验",{"type":150,"tag":151,"props":46320,"children":46321},{},[46322],{"type":155,"value":46323},"最后这条听起来最不起眼，实际经常最残酷。一个漂亮的新 loss 写在论文里只需要半页。一个可靠的 million-token agent rollout system，可能需要几十个人跟各种分布式 bug、CUDA bug、环境 bug、数据 bug 打几个月架。论文不会把那些凌晨三点的 NCCL timeout 写进公式里，但模型能力很可能就死在那里。Kimi K3 的 KDA 为什么重要？不是因为 Linear Attention 本身突然赋予模型神秘推理能力。而是它让 1M context、long-horizon rollout、agentic RL 在计算上变得现实：",{"type":150,"tag":169,"props":46325,"children":46327},{"className":46326},[172],[46328],{"type":150,"tag":169,"props":46329,"children":46331},{"className":46330},[177],[46332,46391],{"type":150,"tag":169,"props":46333,"children":46335},{"className":46334},[182],[46336],{"type":150,"tag":185,"props":46337,"children":46338},{"xmlns":187,"display":188},[46339],{"type":150,"tag":191,"props":46340,"children":46341},{},[46342,46386],{"type":150,"tag":195,"props":46343,"children":46344},{},[46345,46350,46354,46359,46363,46368,46372,46377,46381],{"type":150,"tag":217,"props":46346,"children":46347},{},[46348],{"type":155,"value":46349},"KDA",{"type":150,"tag":417,"props":46351,"children":46352},{},[46353],{"type":155,"value":12954},{"type":150,"tag":217,"props":46355,"children":46356},{},[46357],{"type":155,"value":46358},"长上下文更便宜",{"type":150,"tag":417,"props":46360,"children":46361},{},[46362],{"type":155,"value":12954},{"type":150,"tag":217,"props":46364,"children":46365},{},[46366],{"type":155,"value":46367},"更多 long-horizon RL rollout",{"type":150,"tag":417,"props":46369,"children":46370},{},[46371],{"type":155,"value":12954},{"type":150,"tag":217,"props":46373,"children":46374},{},[46375],{"type":155,"value":46376},"更好的 agent 数据",{"type":150,"tag":417,"props":46378,"children":46379},{},[46380],{"type":155,"value":12954},{"type":150,"tag":217,"props":46382,"children":46383},{},[46384],{"type":155,"value":46385},"Agent 能力提高",{"type":150,"tag":223,"props":46387,"children":46388},{"encoding":225},[46389],{"type":155,"value":46390},"\\text{KDA} \\rightarrow \\text{长上下文更便宜} \\rightarrow \\text{更多 long-horizon RL rollout} \\rightarrow \\text{更好的 agent 数据} \\rightarrow \\text{Agent 能力提高}",{"type":150,"tag":169,"props":46392,"children":46394},{"className":46393,"ariaHidden":214},[232],[46395,46425,46455,46492,46535],{"type":150,"tag":169,"props":46396,"children":46398},{"className":46397},[237],[46399,46403,46412,46416,46421],{"type":150,"tag":169,"props":46400,"children":46402},{"className":46401,"style":5854},[242],[],{"type":150,"tag":169,"props":46404,"children":46406},{"className":46405},[248,155],[46407],{"type":150,"tag":169,"props":46408,"children":46410},{"className":46409},[248],[46411],{"type":155,"value":46349},{"type":150,"tag":169,"props":46413,"children":46415},{"className":46414,"style":839},[558],[],{"type":150,"tag":169,"props":46417,"children":46419},{"className":46418},[844],[46420],{"type":155,"value":12954},{"type":150,"tag":169,"props":46422,"children":46424},{"className":46423,"style":839},[558],[],{"type":150,"tag":169,"props":46426,"children":46428},{"className":46427},[237],[46429,46433,46442,46446,46451],{"type":150,"tag":169,"props":46430,"children":46432},{"className":46431,"style":5854},[242],[],{"type":150,"tag":169,"props":46434,"children":46436},{"className":46435},[248,155],[46437],{"type":150,"tag":169,"props":46438,"children":46440},{"className":46439},[248,297],[46441],{"type":155,"value":46358},{"type":150,"tag":169,"props":46443,"children":46445},{"className":46444,"style":839},[558],[],{"type":150,"tag":169,"props":46447,"children":46449},{"className":46448},[844],[46450],{"type":155,"value":12954},{"type":150,"tag":169,"props":46452,"children":46454},{"className":46453,"style":839},[558],[],{"type":150,"tag":169,"props":46456,"children":46458},{"className":46457},[237],[46459,46463,46479,46483,46488],{"type":150,"tag":169,"props":46460,"children":46462},{"className":46461,"style":1121},[242],[],{"type":150,"tag":169,"props":46464,"children":46466},{"className":46465},[248,155],[46467,46473],{"type":150,"tag":169,"props":46468,"children":46470},{"className":46469},[248,297],[46471],{"type":155,"value":46472},"更多",{"type":150,"tag":169,"props":46474,"children":46476},{"className":46475},[248],[46477],{"type":155,"value":46478}," long-horizon RL rollout",{"type":150,"tag":169,"props":46480,"children":46482},{"className":46481,"style":839},[558],[],{"type":150,"tag":169,"props":46484,"children":46486},{"className":46485},[844],[46487],{"type":155,"value":12954},{"type":150,"tag":169,"props":46489,"children":46491},{"className":46490,"style":839},[558],[],{"type":150,"tag":169,"props":46493,"children":46495},{"className":46494},[237],[46496,46500,46522,46526,46531],{"type":150,"tag":169,"props":46497,"children":46499},{"className":46498,"style":11723},[242],[],{"type":150,"tag":169,"props":46501,"children":46503},{"className":46502},[248,155],[46504,46510,46516],{"type":150,"tag":169,"props":46505,"children":46507},{"className":46506},[248,297],[46508],{"type":155,"value":46509},"更好的",{"type":150,"tag":169,"props":46511,"children":46513},{"className":46512},[248],[46514],{"type":155,"value":46515}," agent ",{"type":150,"tag":169,"props":46517,"children":46519},{"className":46518},[248,297],[46520],{"type":155,"value":46521},"数据",{"type":150,"tag":169,"props":46523,"children":46525},{"className":46524,"style":839},[558],[],{"type":150,"tag":169,"props":46527,"children":46529},{"className":46528},[844],[46530],{"type":155,"value":12954},{"type":150,"tag":169,"props":46532,"children":46534},{"className":46533,"style":839},[558],[],{"type":150,"tag":169,"props":46536,"children":46538},{"className":46537},[237],[46539,46543],{"type":150,"tag":169,"props":46540,"children":46542},{"className":46541,"style":11723},[242],[],{"type":150,"tag":169,"props":46544,"children":46546},{"className":46545},[248,155],[46547,46553],{"type":150,"tag":169,"props":46548,"children":46550},{"className":46549},[248],[46551],{"type":155,"value":46552},"Agent ",{"type":150,"tag":169,"props":46554,"children":46556},{"className":46555},[248,297],[46557],{"type":155,"value":46558},"能力提高",{"type":150,"tag":151,"props":46560,"children":46561},{},[46562],{"type":155,"value":46563},"CSA\u002FHCA 也一样。它最漂亮的地方未必是\"稀疏 attention 本身理解能力比 MLA 强\"，而是 1M context 变得足够便宜。",{"type":150,"tag":169,"props":46565,"children":46567},{"className":46566},[172],[46568],{"type":150,"tag":169,"props":46569,"children":46571},{"className":46570},[177],[46572,46607],{"type":150,"tag":169,"props":46573,"children":46575},{"className":46574},[182],[46576],{"type":150,"tag":185,"props":46577,"children":46578},{"xmlns":187,"display":188},[46579],{"type":150,"tag":191,"props":46580,"children":46581},{},[46582,46602],{"type":150,"tag":195,"props":46583,"children":46584},{},[46585],{"type":150,"tag":199,"props":46586,"children":46587},{"notation":201},[46588],{"type":150,"tag":204,"props":46589,"children":46590},{"scriptlevel":206,"displaystyle":207},[46591],{"type":150,"tag":204,"props":46592,"children":46593},{"scriptlevel":206,"displaystyle":207},[46594],{"type":150,"tag":204,"props":46595,"children":46596},{"scriptlevel":206,"displaystyle":214},[46597],{"type":150,"tag":217,"props":46598,"children":46599},{},[46600],{"type":155,"value":46601},"架构是 compute multiplier，不是 intelligence generator。",{"type":150,"tag":223,"props":46603,"children":46604},{"encoding":225},[46605],{"type":155,"value":46606},"\\boxed{\\text{架构是 compute multiplier，不是 intelligence generator。}}",{"type":150,"tag":169,"props":46608,"children":46610},{"className":46609,"ariaHidden":214},[232],[46611],{"type":150,"tag":169,"props":46612,"children":46614},{"className":46613},[237],[46615,46619],{"type":150,"tag":169,"props":46616,"children":46618},{"className":46617,"style":30651},[242],[],{"type":150,"tag":169,"props":46620,"children":46622},{"className":46621},[248],[46623],{"type":150,"tag":169,"props":46624,"children":46626},{"className":46625},[253,254],[46627,46702],{"type":150,"tag":169,"props":46628,"children":46630},{"className":46629},[259],[46631,46697],{"type":150,"tag":169,"props":46632,"children":46634},{"className":46633,"style":30668},[264],[46635,46686],{"type":150,"tag":169,"props":46636,"children":46637},{"style":30672},[46638,46642],{"type":150,"tag":169,"props":46639,"children":46641},{"className":46640,"style":30677},[274],[],{"type":150,"tag":169,"props":46643,"children":46645},{"className":46644},[280],[46646],{"type":150,"tag":169,"props":46647,"children":46649},{"className":46648},[248],[46650],{"type":150,"tag":169,"props":46651,"children":46653},{"className":46652},[248],[46654],{"type":150,"tag":169,"props":46655,"children":46657},{"className":46656},[248,155],[46658,46664,46670,46675,46681],{"type":150,"tag":169,"props":46659,"children":46661},{"className":46660},[248,297],[46662],{"type":155,"value":46663},"架构是",{"type":150,"tag":169,"props":46665,"children":46667},{"className":46666},[248],[46668],{"type":155,"value":46669}," compute multiplier",{"type":150,"tag":169,"props":46671,"children":46673},{"className":46672},[248,297],[46674],{"type":155,"value":35247},{"type":150,"tag":169,"props":46676,"children":46678},{"className":46677},[248],[46679],{"type":155,"value":46680}," intelligence generator",{"type":150,"tag":169,"props":46682,"children":46684},{"className":46683},[248,297],[46685],{"type":155,"value":904},{"type":150,"tag":169,"props":46687,"children":46688},{"style":30721},[46689,46693],{"type":150,"tag":169,"props":46690,"children":46692},{"className":46691,"style":30677},[274],[],{"type":150,"tag":169,"props":46694,"children":46696},{"className":46695,"style":30730},[348,349],[],{"type":150,"tag":169,"props":46698,"children":46700},{"className":46699},[355],[46701],{"type":155,"value":358},{"type":150,"tag":169,"props":46703,"children":46705},{"className":46704},[259],[46706],{"type":150,"tag":169,"props":46707,"children":46709},{"className":46708,"style":366},[264],[46710],{"type":150,"tag":169,"props":46711,"children":46712},{},[],{"type":150,"tag":372,"props":46714,"children":46716},{"id":46715},"算力预算",[46717],{"type":155,"value":46715},{"type":150,"tag":151,"props":46719,"children":46720},{},[46721],{"type":155,"value":46722},"如果架构主要在省钱，那\"该不该冒架构的险\"就变成了一个纯粹的预算问题。假设已验证的 Transformer scaling recipe 能拿到 95 分。新架构理论上能到 100，但有 20% 概率训练炸掉，只剩 70。如果算力富裕，为什么要冒险？把已验证的架构再 scale 10 倍，可能直接拿到 98，而且更可靠。",{"type":150,"tag":169,"props":46724,"children":46726},{"className":46725},[172],[46727],{"type":150,"tag":169,"props":46728,"children":46730},{"className":46729},[177],[46731,46766],{"type":150,"tag":169,"props":46732,"children":46734},{"className":46733},[182],[46735],{"type":150,"tag":185,"props":46736,"children":46737},{"xmlns":187,"display":188},[46738],{"type":150,"tag":191,"props":46739,"children":46740},{},[46741,46761],{"type":150,"tag":195,"props":46742,"children":46743},{},[46744],{"type":150,"tag":199,"props":46745,"children":46746},{"notation":201},[46747],{"type":150,"tag":204,"props":46748,"children":46749},{"scriptlevel":206,"displaystyle":207},[46750],{"type":150,"tag":204,"props":46751,"children":46752},{"scriptlevel":206,"displaystyle":207},[46753],{"type":150,"tag":204,"props":46754,"children":46755},{"scriptlevel":206,"displaystyle":214},[46756],{"type":150,"tag":217,"props":46757,"children":46758},{},[46759],{"type":155,"value":46760},"scale proven architecture first",{"type":150,"tag":223,"props":46762,"children":46763},{"encoding":225},[46764],{"type":155,"value":46765},"\\boxed{\\text{scale proven architecture first}}",{"type":150,"tag":169,"props":46767,"children":46769},{"className":46768,"ariaHidden":214},[232],[46770],{"type":150,"tag":169,"props":46771,"children":46773},{"className":46772},[237],[46774,46778],{"type":150,"tag":169,"props":46775,"children":46777},{"className":46776,"style":30651},[242],[],{"type":150,"tag":169,"props":46779,"children":46781},{"className":46780},[248],[46782],{"type":150,"tag":169,"props":46783,"children":46785},{"className":46784},[253,254],[46786,46838],{"type":150,"tag":169,"props":46787,"children":46789},{"className":46788},[259],[46790,46833],{"type":150,"tag":169,"props":46791,"children":46793},{"className":46792,"style":30668},[264],[46794,46822],{"type":150,"tag":169,"props":46795,"children":46796},{"style":30672},[46797,46801],{"type":150,"tag":169,"props":46798,"children":46800},{"className":46799,"style":30677},[274],[],{"type":150,"tag":169,"props":46802,"children":46804},{"className":46803},[280],[46805],{"type":150,"tag":169,"props":46806,"children":46808},{"className":46807},[248],[46809],{"type":150,"tag":169,"props":46810,"children":46812},{"className":46811},[248],[46813],{"type":150,"tag":169,"props":46814,"children":46816},{"className":46815},[248,155],[46817],{"type":150,"tag":169,"props":46818,"children":46820},{"className":46819},[248],[46821],{"type":155,"value":46760},{"type":150,"tag":169,"props":46823,"children":46824},{"style":30721},[46825,46829],{"type":150,"tag":169,"props":46826,"children":46828},{"className":46827,"style":30677},[274],[],{"type":150,"tag":169,"props":46830,"children":46832},{"className":46831,"style":30730},[348,349],[],{"type":150,"tag":169,"props":46834,"children":46836},{"className":46835},[355],[46837],{"type":155,"value":358},{"type":150,"tag":169,"props":46839,"children":46841},{"className":46840},[259],[46842],{"type":150,"tag":169,"props":46843,"children":46845},{"className":46844,"style":366},[264],[46846],{"type":150,"tag":169,"props":46847,"children":46848},{},[],{"type":150,"tag":151,"props":46850,"children":46851},{},[46852],{"type":155,"value":46853},"这其实是一种\"富人的保守\"。而算力紧张的实验室没有 10× compute 可以挥霍，只能想办法：",{"type":150,"tag":169,"props":46855,"children":46857},{"className":46856},[172],[46858],{"type":150,"tag":169,"props":46859,"children":46861},{"className":46860},[177],[46862,46927],{"type":150,"tag":169,"props":46863,"children":46865},{"className":46864},[182],[46866],{"type":150,"tag":185,"props":46867,"children":46868},{"xmlns":187,"display":188},[46869],{"type":150,"tag":191,"props":46870,"children":46871},{},[46872,46922],{"type":150,"tag":195,"props":46873,"children":46874},{},[46875],{"type":150,"tag":199,"props":46876,"children":46877},{"notation":201},[46878],{"type":150,"tag":204,"props":46879,"children":46880},{"scriptlevel":206,"displaystyle":207},[46881],{"type":150,"tag":204,"props":46882,"children":46883},{"scriptlevel":206,"displaystyle":207},[46884],{"type":150,"tag":204,"props":46885,"children":46886},{"scriptlevel":206,"displaystyle":214},[46887],{"type":150,"tag":195,"props":46888,"children":46889},{},[46890,46895,46899,46903,46908,46913,46917],{"type":150,"tag":217,"props":46891,"children":46892},{},[46893],{"type":155,"value":46894},"让 ",{"type":150,"tag":411,"props":46896,"children":46897},{},[46898],{"type":155,"value":415},{"type":150,"tag":417,"props":46900,"children":46901},{},[46902],{"type":155,"value":5554},{"type":150,"tag":217,"props":46904,"children":46905},{},[46906],{"type":155,"value":46907}," compute 做出 ",{"type":150,"tag":411,"props":46909,"children":46910},{},[46911],{"type":155,"value":46912},"3",{"type":150,"tag":417,"props":46914,"children":46915},{},[46916],{"type":155,"value":5554},{"type":150,"tag":217,"props":46918,"children":46919},{},[46920],{"type":155,"value":46921}," compute 的效果",{"type":150,"tag":223,"props":46923,"children":46924},{"encoding":225},[46925],{"type":155,"value":46926},"\\boxed{\\text{让 } 1\\times \\text{ compute 做出 } 3\\times \\text{ compute 的效果}}",{"type":150,"tag":169,"props":46928,"children":46930},{"className":46929,"ariaHidden":214},[232],[46931],{"type":150,"tag":169,"props":46932,"children":46934},{"className":46933},[237],[46935,46939],{"type":150,"tag":169,"props":46936,"children":46938},{"className":46937,"style":243},[242],[],{"type":150,"tag":169,"props":46940,"children":46942},{"className":46941},[248],[46943],{"type":150,"tag":169,"props":46944,"children":46946},{"className":46945},[253,254],[46947,47076],{"type":150,"tag":169,"props":46948,"children":46950},{"className":46949},[259],[46951,47071],{"type":150,"tag":169,"props":46952,"children":46954},{"className":46953,"style":265},[264],[46955,47060],{"type":150,"tag":169,"props":46956,"children":46957},{"style":269},[46958,46962],{"type":150,"tag":169,"props":46959,"children":46961},{"className":46960,"style":275},[274],[],{"type":150,"tag":169,"props":46963,"children":46965},{"className":46964},[280],[46966],{"type":150,"tag":169,"props":46967,"children":46969},{"className":46968},[248],[46970],{"type":150,"tag":169,"props":46971,"children":46973},{"className":46972},[248],[46974,46989,46994,46998,47003,47007,47027,47032,47036,47041,47045],{"type":150,"tag":169,"props":46975,"children":46977},{"className":46976},[248,155],[46978,46984],{"type":150,"tag":169,"props":46979,"children":46981},{"className":46980},[248,297],[46982],{"type":155,"value":46983},"让",{"type":150,"tag":169,"props":46985,"children":46987},{"className":46986},[248],[46988],{"type":155,"value":8763},{"type":150,"tag":169,"props":46990,"children":46992},{"className":46991},[248],[46993],{"type":155,"value":415},{"type":150,"tag":169,"props":46995,"children":46997},{"className":46996,"style":5167},[558],[],{"type":150,"tag":169,"props":46999,"children":47001},{"className":47000},[5172],[47002],{"type":155,"value":5554},{"type":150,"tag":169,"props":47004,"children":47006},{"className":47005,"style":5167},[558],[],{"type":150,"tag":169,"props":47008,"children":47010},{"className":47009},[248,155],[47011,47016,47022],{"type":150,"tag":169,"props":47012,"children":47014},{"className":47013},[248],[47015],{"type":155,"value":330},{"type":150,"tag":169,"props":47017,"children":47019},{"className":47018},[248,297],[47020],{"type":155,"value":47021},"做出",{"type":150,"tag":169,"props":47023,"children":47025},{"className":47024},[248],[47026],{"type":155,"value":8763},{"type":150,"tag":169,"props":47028,"children":47030},{"className":47029},[248],[47031],{"type":155,"value":46912},{"type":150,"tag":169,"props":47033,"children":47035},{"className":47034,"style":5167},[558],[],{"type":150,"tag":169,"props":47037,"children":47039},{"className":47038},[5172],[47040],{"type":155,"value":5554},{"type":150,"tag":169,"props":47042,"children":47044},{"className":47043,"style":5167},[558],[],{"type":150,"tag":169,"props":47046,"children":47048},{"className":47047},[248,155],[47049,47054],{"type":150,"tag":169,"props":47050,"children":47052},{"className":47051},[248],[47053],{"type":155,"value":330},{"type":150,"tag":169,"props":47055,"children":47057},{"className":47056},[248,297],[47058],{"type":155,"value":47059},"的效果",{"type":150,"tag":169,"props":47061,"children":47062},{"style":339},[47063,47067],{"type":150,"tag":169,"props":47064,"children":47066},{"className":47065,"style":275},[274],[],{"type":150,"tag":169,"props":47068,"children":47070},{"className":47069,"style":350},[348,349],[],{"type":150,"tag":169,"props":47072,"children":47074},{"className":47073},[355],[47075],{"type":155,"value":358},{"type":150,"tag":169,"props":47077,"children":47079},{"className":47078},[259],[47080],{"type":150,"tag":169,"props":47081,"children":47083},{"className":47082,"style":366},[264],[47084],{"type":150,"tag":169,"props":47085,"children":47086},{},[],{"type":150,"tag":151,"props":47088,"children":47089},{},[47090],{"type":155,"value":47091},"于是架构创新的 ROI 突然变得非常高。从优化目标看更直白。算力富裕的一方更接近：",{"type":150,"tag":169,"props":47093,"children":47095},{"className":47094},[172],[47096],{"type":150,"tag":169,"props":47097,"children":47099},{"className":47098},[177],[47100,47142],{"type":150,"tag":169,"props":47101,"children":47103},{"className":47102},[182],[47104],{"type":150,"tag":185,"props":47105,"children":47106},{"xmlns":187,"display":188},[47107],{"type":150,"tag":191,"props":47108,"children":47109},{},[47110,47137],{"type":150,"tag":195,"props":47111,"children":47112},{},[47113,47117,47121,47125,47129,47132],{"type":150,"tag":405,"props":47114,"children":47115},{},[47116],{"type":155,"value":42968},{"type":150,"tag":417,"props":47118,"children":47119},{},[47120],{"type":155,"value":1991},{"type":150,"tag":217,"props":47122,"children":47123},{},[47124],{"type":155,"value":45673},{"type":150,"tag":417,"props":47126,"children":47127},{"separator":214},[47128],{"type":155,"value":421},{"type":150,"tag":558,"props":47130,"children":47131},{"width":3491},[],{"type":150,"tag":217,"props":47133,"children":47134},{},[47135],{"type":155,"value":47136},"s.t. Compute 很大",{"type":150,"tag":223,"props":47138,"children":47139},{"encoding":225},[47140],{"type":155,"value":47141},"\\max \\text{Capability}, \\quad \\text{s.t. Compute 很大}",{"type":150,"tag":169,"props":47143,"children":47145},{"className":47144,"ariaHidden":214},[232],[47146],{"type":150,"tag":169,"props":47147,"children":47149},{"className":47148},[237],[47150,47154,47159,47163,47172,47177,47181,47185],{"type":150,"tag":169,"props":47151,"children":47153},{"className":47152,"style":1121},[242],[],{"type":150,"tag":169,"props":47155,"children":47157},{"className":47156},[2695],[47158],{"type":155,"value":42968},{"type":150,"tag":169,"props":47160,"children":47162},{"className":47161,"style":559},[558],[],{"type":150,"tag":169,"props":47164,"children":47166},{"className":47165},[248,155],[47167],{"type":150,"tag":169,"props":47168,"children":47170},{"className":47169},[248],[47171],{"type":155,"value":45673},{"type":150,"tag":169,"props":47173,"children":47175},{"className":47174},[552],[47176],{"type":155,"value":421},{"type":150,"tag":169,"props":47178,"children":47180},{"className":47179,"style":4087},[558],[],{"type":150,"tag":169,"props":47182,"children":47184},{"className":47183,"style":559},[558],[],{"type":150,"tag":169,"props":47186,"children":47188},{"className":47187},[248,155],[47189,47195],{"type":150,"tag":169,"props":47190,"children":47192},{"className":47191},[248],[47193],{"type":155,"value":47194},"s.t. Compute ",{"type":150,"tag":169,"props":47196,"children":47198},{"className":47197},[248,297],[47199],{"type":155,"value":47200},"很大",{"type":150,"tag":151,"props":47202,"children":47203},{},[47204],{"type":155,"value":47205},"所以可以接受较低的 architecture efficiency，只要 scaling 稳。而算力受限的一方更像：",{"type":150,"tag":169,"props":47207,"children":47209},{"className":47208},[172],[47210],{"type":150,"tag":169,"props":47211,"children":47213},{"className":47212},[177],[47214,47252],{"type":150,"tag":169,"props":47215,"children":47217},{"className":47216},[182],[47218],{"type":150,"tag":185,"props":47219,"children":47220},{"xmlns":187,"display":188},[47221],{"type":150,"tag":191,"props":47222,"children":47223},{},[47224,47247],{"type":150,"tag":195,"props":47225,"children":47226},{},[47227,47231,47235],{"type":150,"tag":405,"props":47228,"children":47229},{},[47230],{"type":155,"value":42968},{"type":150,"tag":417,"props":47232,"children":47233},{},[47234],{"type":155,"value":1991},{"type":150,"tag":1952,"props":47236,"children":47237},{},[47238,47242],{"type":150,"tag":217,"props":47239,"children":47240},{},[47241],{"type":155,"value":45673},{"type":150,"tag":217,"props":47243,"children":47244},{},[47245],{"type":155,"value":47246},"Compute",{"type":150,"tag":223,"props":47248,"children":47249},{"encoding":225},[47250],{"type":155,"value":47251},"\\max \\frac{\\text{Capability}}{\\text{Compute}}",{"type":150,"tag":169,"props":47253,"children":47255},{"className":47254,"ariaHidden":214},[232],[47256],{"type":150,"tag":169,"props":47257,"children":47259},{"className":47258},[237],[47260,47265,47270,47274],{"type":150,"tag":169,"props":47261,"children":47264},{"className":47262,"style":47263},[242],"height:2.2519em;vertical-align:-0.8804em;",[],{"type":150,"tag":169,"props":47266,"children":47268},{"className":47267},[2695],[47269],{"type":155,"value":42968},{"type":150,"tag":169,"props":47271,"children":47273},{"className":47272,"style":559},[558],[],{"type":150,"tag":169,"props":47275,"children":47277},{"className":47276},[248],[47278,47282,47367],{"type":150,"tag":169,"props":47279,"children":47281},{"className":47280},[1004,2660],[],{"type":150,"tag":169,"props":47283,"children":47285},{"className":47284},[1952],[47286],{"type":150,"tag":169,"props":47287,"children":47289},{"className":47288},[253,254],[47290,47355],{"type":150,"tag":169,"props":47291,"children":47293},{"className":47292},[259],[47294,47350],{"type":150,"tag":169,"props":47295,"children":47298},{"className":47296,"style":47297},[264],"height:1.3714em;",[47299,47319,47330],{"type":150,"tag":169,"props":47300,"children":47301},{"style":8422},[47302,47306],{"type":150,"tag":169,"props":47303,"children":47305},{"className":47304,"style":8427},[274],[],{"type":150,"tag":169,"props":47307,"children":47309},{"className":47308},[248],[47310],{"type":150,"tag":169,"props":47311,"children":47313},{"className":47312},[248,155],[47314],{"type":150,"tag":169,"props":47315,"children":47317},{"className":47316},[248],[47318],{"type":155,"value":47246},{"type":150,"tag":169,"props":47320,"children":47321},{"style":8440},[47322,47326],{"type":150,"tag":169,"props":47323,"children":47325},{"className":47324,"style":8427},[274],[],{"type":150,"tag":169,"props":47327,"children":47329},{"className":47328,"style":2982},[2981],[],{"type":150,"tag":169,"props":47331,"children":47332},{"style":8452},[47333,47337],{"type":150,"tag":169,"props":47334,"children":47336},{"className":47335,"style":8427},[274],[],{"type":150,"tag":169,"props":47338,"children":47340},{"className":47339},[248],[47341],{"type":150,"tag":169,"props":47342,"children":47344},{"className":47343},[248,155],[47345],{"type":150,"tag":169,"props":47346,"children":47348},{"className":47347},[248],[47349],{"type":155,"value":45673},{"type":150,"tag":169,"props":47351,"children":47353},{"className":47352},[355],[47354],{"type":155,"value":358},{"type":150,"tag":169,"props":47356,"children":47358},{"className":47357},[259],[47359],{"type":150,"tag":169,"props":47360,"children":47363},{"className":47361,"style":47362},[264],"height:0.8804em;",[47364],{"type":150,"tag":169,"props":47365,"children":47366},{},[],{"type":150,"tag":169,"props":47368,"children":47370},{"className":47369},[1015,2660],[],{"type":150,"tag":151,"props":47372,"children":47373},{},[47374],{"type":155,"value":47375},"于是被逼出 MLA、MoE、Sparse、Linear、各种 compression。这和芯片行业很像。资源充裕时加晶体管，资源受限时改 architecture。所以很多所谓架构路线之争，底层其实是不同算力预算下的经济学。把这件事再拆细一点，会发现竞争其实发生在四种资本上：",{"type":150,"tag":169,"props":47377,"children":47379},{"className":47378},[172],[47380],{"type":150,"tag":169,"props":47381,"children":47383},{"className":47382},[177],[47384,47419],{"type":150,"tag":169,"props":47385,"children":47387},{"className":47386},[182],[47388],{"type":150,"tag":185,"props":47389,"children":47390},{"xmlns":187,"display":188},[47391],{"type":150,"tag":191,"props":47392,"children":47393},{},[47394,47414],{"type":150,"tag":195,"props":47395,"children":47396},{},[47397],{"type":150,"tag":199,"props":47398,"children":47399},{"notation":201},[47400],{"type":150,"tag":204,"props":47401,"children":47402},{"scriptlevel":206,"displaystyle":207},[47403],{"type":150,"tag":204,"props":47404,"children":47405},{"scriptlevel":206,"displaystyle":207},[47406],{"type":150,"tag":204,"props":47407,"children":47408},{"scriptlevel":206,"displaystyle":214},[47409],{"type":150,"tag":217,"props":47410,"children":47411},{},[47412],{"type":155,"value":47413},"Compute Capital",{"type":150,"tag":223,"props":47415,"children":47416},{"encoding":225},[47417],{"type":155,"value":47418},"\\boxed{\\text{Compute Capital}}",{"type":150,"tag":169,"props":47420,"children":47422},{"className":47421,"ariaHidden":214},[232],[47423],{"type":150,"tag":169,"props":47424,"children":47426},{"className":47425},[237],[47427,47431],{"type":150,"tag":169,"props":47428,"children":47430},{"className":47429,"style":30651},[242],[],{"type":150,"tag":169,"props":47432,"children":47434},{"className":47433},[248],[47435],{"type":150,"tag":169,"props":47436,"children":47438},{"className":47437},[253,254],[47439,47491],{"type":150,"tag":169,"props":47440,"children":47442},{"className":47441},[259],[47443,47486],{"type":150,"tag":169,"props":47444,"children":47446},{"className":47445,"style":30668},[264],[47447,47475],{"type":150,"tag":169,"props":47448,"children":47449},{"style":30672},[47450,47454],{"type":150,"tag":169,"props":47451,"children":47453},{"className":47452,"style":30677},[274],[],{"type":150,"tag":169,"props":47455,"children":47457},{"className":47456},[280],[47458],{"type":150,"tag":169,"props":47459,"children":47461},{"className":47460},[248],[47462],{"type":150,"tag":169,"props":47463,"children":47465},{"className":47464},[248],[47466],{"type":150,"tag":169,"props":47467,"children":47469},{"className":47468},[248,155],[47470],{"type":150,"tag":169,"props":47471,"children":47473},{"className":47472},[248],[47474],{"type":155,"value":47413},{"type":150,"tag":169,"props":47476,"children":47477},{"style":30721},[47478,47482],{"type":150,"tag":169,"props":47479,"children":47481},{"className":47480,"style":30677},[274],[],{"type":150,"tag":169,"props":47483,"children":47485},{"className":47484,"style":30730},[348,349],[],{"type":150,"tag":169,"props":47487,"children":47489},{"className":47488},[355],[47490],{"type":155,"value":358},{"type":150,"tag":169,"props":47492,"children":47494},{"className":47493},[259],[47495],{"type":150,"tag":169,"props":47496,"children":47498},{"className":47497,"style":366},[264],[47499],{"type":150,"tag":169,"props":47500,"children":47501},{},[],{"type":150,"tag":151,"props":47503,"children":47504},{},[47505],{"type":155,"value":47506},"GPU、HBM、network、datacenter。",{"type":150,"tag":169,"props":47508,"children":47510},{"className":47509},[172],[47511],{"type":150,"tag":169,"props":47512,"children":47514},{"className":47513},[177],[47515,47550],{"type":150,"tag":169,"props":47516,"children":47518},{"className":47517},[182],[47519],{"type":150,"tag":185,"props":47520,"children":47521},{"xmlns":187,"display":188},[47522],{"type":150,"tag":191,"props":47523,"children":47524},{},[47525,47545],{"type":150,"tag":195,"props":47526,"children":47527},{},[47528],{"type":150,"tag":199,"props":47529,"children":47530},{"notation":201},[47531],{"type":150,"tag":204,"props":47532,"children":47533},{"scriptlevel":206,"displaystyle":207},[47534],{"type":150,"tag":204,"props":47535,"children":47536},{"scriptlevel":206,"displaystyle":207},[47537],{"type":150,"tag":204,"props":47538,"children":47539},{"scriptlevel":206,"displaystyle":214},[47540],{"type":150,"tag":217,"props":47541,"children":47542},{},[47543],{"type":155,"value":47544},"Data Capital",{"type":150,"tag":223,"props":47546,"children":47547},{"encoding":225},[47548],{"type":155,"value":47549},"\\boxed{\\text{Data Capital}}",{"type":150,"tag":169,"props":47551,"children":47553},{"className":47552,"ariaHidden":214},[232],[47554],{"type":150,"tag":169,"props":47555,"children":47557},{"className":47556},[237],[47558,47562],{"type":150,"tag":169,"props":47559,"children":47561},{"className":47560,"style":30651},[242],[],{"type":150,"tag":169,"props":47563,"children":47565},{"className":47564},[248],[47566],{"type":150,"tag":169,"props":47567,"children":47569},{"className":47568},[253,254],[47570,47622],{"type":150,"tag":169,"props":47571,"children":47573},{"className":47572},[259],[47574,47617],{"type":150,"tag":169,"props":47575,"children":47577},{"className":47576,"style":30668},[264],[47578,47606],{"type":150,"tag":169,"props":47579,"children":47580},{"style":30672},[47581,47585],{"type":150,"tag":169,"props":47582,"children":47584},{"className":47583,"style":30677},[274],[],{"type":150,"tag":169,"props":47586,"children":47588},{"className":47587},[280],[47589],{"type":150,"tag":169,"props":47590,"children":47592},{"className":47591},[248],[47593],{"type":150,"tag":169,"props":47594,"children":47596},{"className":47595},[248],[47597],{"type":150,"tag":169,"props":47598,"children":47600},{"className":47599},[248,155],[47601],{"type":150,"tag":169,"props":47602,"children":47604},{"className":47603},[248],[47605],{"type":155,"value":47544},{"type":150,"tag":169,"props":47607,"children":47608},{"style":30721},[47609,47613],{"type":150,"tag":169,"props":47610,"children":47612},{"className":47611,"style":30677},[274],[],{"type":150,"tag":169,"props":47614,"children":47616},{"className":47615,"style":30730},[348,349],[],{"type":150,"tag":169,"props":47618,"children":47620},{"className":47619},[355],[47621],{"type":155,"value":358},{"type":150,"tag":169,"props":47623,"children":47625},{"className":47624},[259],[47626],{"type":150,"tag":169,"props":47627,"children":47629},{"className":47628,"style":366},[264],[47630],{"type":150,"tag":169,"props":47631,"children":47632},{},[],{"type":150,"tag":151,"props":47634,"children":47635},{},[47636],{"type":155,"value":47637},"真实用户、agent trajectories、enterprise workflow、feedback。",{"type":150,"tag":169,"props":47639,"children":47641},{"className":47640},[172],[47642],{"type":150,"tag":169,"props":47643,"children":47645},{"className":47644},[177],[47646,47681],{"type":150,"tag":169,"props":47647,"children":47649},{"className":47648},[182],[47650],{"type":150,"tag":185,"props":47651,"children":47652},{"xmlns":187,"display":188},[47653],{"type":150,"tag":191,"props":47654,"children":47655},{},[47656,47676],{"type":150,"tag":195,"props":47657,"children":47658},{},[47659],{"type":150,"tag":199,"props":47660,"children":47661},{"notation":201},[47662],{"type":150,"tag":204,"props":47663,"children":47664},{"scriptlevel":206,"displaystyle":207},[47665],{"type":150,"tag":204,"props":47666,"children":47667},{"scriptlevel":206,"displaystyle":207},[47668],{"type":150,"tag":204,"props":47669,"children":47670},{"scriptlevel":206,"displaystyle":214},[47671],{"type":150,"tag":217,"props":47672,"children":47673},{},[47674],{"type":155,"value":47675},"Algorithmic Capital",{"type":150,"tag":223,"props":47677,"children":47678},{"encoding":225},[47679],{"type":155,"value":47680},"\\boxed{\\text{Algorithmic Capital}}",{"type":150,"tag":169,"props":47682,"children":47684},{"className":47683,"ariaHidden":214},[232],[47685],{"type":150,"tag":169,"props":47686,"children":47688},{"className":47687},[237],[47689,47693],{"type":150,"tag":169,"props":47690,"children":47692},{"className":47691,"style":30651},[242],[],{"type":150,"tag":169,"props":47694,"children":47696},{"className":47695},[248],[47697],{"type":150,"tag":169,"props":47698,"children":47700},{"className":47699},[253,254],[47701,47753],{"type":150,"tag":169,"props":47702,"children":47704},{"className":47703},[259],[47705,47748],{"type":150,"tag":169,"props":47706,"children":47708},{"className":47707,"style":30668},[264],[47709,47737],{"type":150,"tag":169,"props":47710,"children":47711},{"style":30672},[47712,47716],{"type":150,"tag":169,"props":47713,"children":47715},{"className":47714,"style":30677},[274],[],{"type":150,"tag":169,"props":47717,"children":47719},{"className":47718},[280],[47720],{"type":150,"tag":169,"props":47721,"children":47723},{"className":47722},[248],[47724],{"type":150,"tag":169,"props":47725,"children":47727},{"className":47726},[248],[47728],{"type":150,"tag":169,"props":47729,"children":47731},{"className":47730},[248,155],[47732],{"type":150,"tag":169,"props":47733,"children":47735},{"className":47734},[248],[47736],{"type":155,"value":47675},{"type":150,"tag":169,"props":47738,"children":47739},{"style":30721},[47740,47744],{"type":150,"tag":169,"props":47741,"children":47743},{"className":47742,"style":30677},[274],[],{"type":150,"tag":169,"props":47745,"children":47747},{"className":47746,"style":30730},[348,349],[],{"type":150,"tag":169,"props":47749,"children":47751},{"className":47750},[355],[47752],{"type":155,"value":358},{"type":150,"tag":169,"props":47754,"children":47756},{"className":47755},[259],[47757],{"type":150,"tag":169,"props":47758,"children":47760},{"className":47759,"style":366},[264],[47761],{"type":150,"tag":169,"props":47762,"children":47763},{},[],{"type":150,"tag":151,"props":47765,"children":47766},{},[47767],{"type":155,"value":47768},"architecture、training recipe、RL、verifier。",{"type":150,"tag":169,"props":47770,"children":47772},{"className":47771},[172],[47773],{"type":150,"tag":169,"props":47774,"children":47776},{"className":47775},[177],[47777,47812],{"type":150,"tag":169,"props":47778,"children":47780},{"className":47779},[182],[47781],{"type":150,"tag":185,"props":47782,"children":47783},{"xmlns":187,"display":188},[47784],{"type":150,"tag":191,"props":47785,"children":47786},{},[47787,47807],{"type":150,"tag":195,"props":47788,"children":47789},{},[47790],{"type":150,"tag":199,"props":47791,"children":47792},{"notation":201},[47793],{"type":150,"tag":204,"props":47794,"children":47795},{"scriptlevel":206,"displaystyle":207},[47796],{"type":150,"tag":204,"props":47797,"children":47798},{"scriptlevel":206,"displaystyle":207},[47799],{"type":150,"tag":204,"props":47800,"children":47801},{"scriptlevel":206,"displaystyle":214},[47802],{"type":150,"tag":217,"props":47803,"children":47804},{},[47805],{"type":155,"value":47806},"Infrastructure Capital",{"type":150,"tag":223,"props":47808,"children":47809},{"encoding":225},[47810],{"type":155,"value":47811},"\\boxed{\\text{Infrastructure Capital}}",{"type":150,"tag":169,"props":47813,"children":47815},{"className":47814,"ariaHidden":214},[232],[47816],{"type":150,"tag":169,"props":47817,"children":47819},{"className":47818},[237],[47820,47824],{"type":150,"tag":169,"props":47821,"children":47823},{"className":47822,"style":30651},[242],[],{"type":150,"tag":169,"props":47825,"children":47827},{"className":47826},[248],[47828],{"type":150,"tag":169,"props":47829,"children":47831},{"className":47830},[253,254],[47832,47884],{"type":150,"tag":169,"props":47833,"children":47835},{"className":47834},[259],[47836,47879],{"type":150,"tag":169,"props":47837,"children":47839},{"className":47838,"style":30668},[264],[47840,47868],{"type":150,"tag":169,"props":47841,"children":47842},{"style":30672},[47843,47847],{"type":150,"tag":169,"props":47844,"children":47846},{"className":47845,"style":30677},[274],[],{"type":150,"tag":169,"props":47848,"children":47850},{"className":47849},[280],[47851],{"type":150,"tag":169,"props":47852,"children":47854},{"className":47853},[248],[47855],{"type":150,"tag":169,"props":47856,"children":47858},{"className":47857},[248],[47859],{"type":150,"tag":169,"props":47860,"children":47862},{"className":47861},[248,155],[47863],{"type":150,"tag":169,"props":47864,"children":47866},{"className":47865},[248],[47867],{"type":155,"value":47806},{"type":150,"tag":169,"props":47869,"children":47870},{"style":30721},[47871,47875],{"type":150,"tag":169,"props":47872,"children":47874},{"className":47873,"style":30677},[274],[],{"type":150,"tag":169,"props":47876,"children":47878},{"className":47877,"style":30730},[348,349],[],{"type":150,"tag":169,"props":47880,"children":47882},{"className":47881},[355],[47883],{"type":155,"value":358},{"type":150,"tag":169,"props":47885,"children":47887},{"className":47886},[259],[47888],{"type":150,"tag":169,"props":47889,"children":47891},{"className":47890,"style":366},[264],[47892],{"type":150,"tag":169,"props":47893,"children":47894},{},[],{"type":150,"tag":151,"props":47896,"children":47897},{},[47898,47900],{"type":155,"value":47899},"稳定地把几千几万张 GPU、RL environment、rollout 全跑起来。算力富裕的一方在前两项和第四项上有巨大积累；算力受限的一方被迫在第三项特别凶。这也是为什么\"美国靠暴力 scaling、中国靠聪明架构\"这种说法太简化了。更接近事实的写法是：",{"type":150,"tag":158,"props":47901,"children":47902},{},[47903],{"type":155,"value":47904},"一边在用资源优势降低研究风险，另一边在用资源约束提高创新收益。",{"type":150,"tag":151,"props":47906,"children":47907},{},[47908],{"type":155,"value":47909},"而且\"数据\"这一项也没法简单说成谁多谁少。真正稀缺的早就不是互联网文本了。Wikipedia、Common Crawl、GitHub 这些 public corpus，大家拿到的量级差不多。现在值钱的是高质量 reasoning traces、真实 agent trajectories、用户交互反馈、专业任务数据、tool-use traces、可验证环境。这些东西不是爬虫能给你的，是产品规模给的。",{"type":150,"tag":151,"props":47911,"children":47912},{},[47913],{"type":155,"value":47914},"所以更要命的不是\"谁有更多互联网 token\"，而是\"谁有更多高价值的 interaction token\"。但这里有个反转。",{"type":150,"tag":151,"props":47916,"children":47917},{},[47918],{"type":155,"value":47919},"一旦某个架构优化被证明有效，情况就变了。DeepSeek 做出 MLA，或者 Kimi 做出成熟的 Linear Attention，算力多的一方并不会因为自己算力多就拒绝使用。它们完全可以：",{"type":150,"tag":169,"props":47921,"children":47923},{"className":47922},[172],[47924],{"type":150,"tag":169,"props":47925,"children":47927},{"className":47926},[177],[47928,47975],{"type":150,"tag":169,"props":47929,"children":47931},{"className":47930},[182],[47932],{"type":150,"tag":185,"props":47933,"children":47934},{"xmlns":187,"display":188},[47935],{"type":150,"tag":191,"props":47936,"children":47937},{},[47938,47970],{"type":150,"tag":195,"props":47939,"children":47940},{},[47941],{"type":150,"tag":199,"props":47942,"children":47943},{"notation":201},[47944],{"type":150,"tag":204,"props":47945,"children":47946},{"scriptlevel":206,"displaystyle":207},[47947],{"type":150,"tag":204,"props":47948,"children":47949},{"scriptlevel":206,"displaystyle":207},[47950],{"type":150,"tag":204,"props":47951,"children":47952},{"scriptlevel":206,"displaystyle":214},[47953],{"type":150,"tag":195,"props":47954,"children":47955},{},[47956,47961,47965],{"type":150,"tag":217,"props":47957,"children":47958},{},[47959],{"type":155,"value":47960},"更多算力",{"type":150,"tag":417,"props":47962,"children":47963},{},[47964],{"type":155,"value":5554},{"type":150,"tag":217,"props":47966,"children":47967},{},[47968],{"type":155,"value":47969},"更高效架构",{"type":150,"tag":223,"props":47971,"children":47972},{"encoding":225},[47973],{"type":155,"value":47974},"\\boxed{\\text{更多算力} \\times \\text{更高效架构}}",{"type":150,"tag":169,"props":47976,"children":47978},{"className":47977,"ariaHidden":214},[232],[47979],{"type":150,"tag":169,"props":47980,"children":47982},{"className":47981},[237],[47983,47987],{"type":150,"tag":169,"props":47984,"children":47986},{"className":47985,"style":35664},[242],[],{"type":150,"tag":169,"props":47988,"children":47990},{"className":47989},[248],[47991],{"type":150,"tag":169,"props":47992,"children":47994},{"className":47993},[253,254],[47995,48069],{"type":150,"tag":169,"props":47996,"children":47998},{"className":47997},[259],[47999,48064],{"type":150,"tag":169,"props":48000,"children":48002},{"className":48001,"style":265},[264],[48003,48053],{"type":150,"tag":169,"props":48004,"children":48005},{"style":35684},[48006,48010],{"type":150,"tag":169,"props":48007,"children":48009},{"className":48008,"style":35689},[274],[],{"type":150,"tag":169,"props":48011,"children":48013},{"className":48012},[280],[48014],{"type":150,"tag":169,"props":48015,"children":48017},{"className":48016},[248],[48018],{"type":150,"tag":169,"props":48019,"children":48021},{"className":48020},[248],[48022,48031,48035,48040,48044],{"type":150,"tag":169,"props":48023,"children":48025},{"className":48024},[248,155],[48026],{"type":150,"tag":169,"props":48027,"children":48029},{"className":48028},[248,297],[48030],{"type":155,"value":47960},{"type":150,"tag":169,"props":48032,"children":48034},{"className":48033,"style":5167},[558],[],{"type":150,"tag":169,"props":48036,"children":48038},{"className":48037},[5172],[48039],{"type":155,"value":5554},{"type":150,"tag":169,"props":48041,"children":48043},{"className":48042,"style":5167},[558],[],{"type":150,"tag":169,"props":48045,"children":48047},{"className":48046},[248,155],[48048],{"type":150,"tag":169,"props":48049,"children":48051},{"className":48050},[248,297],[48052],{"type":155,"value":47969},{"type":150,"tag":169,"props":48054,"children":48055},{"style":339},[48056,48060],{"type":150,"tag":169,"props":48057,"children":48059},{"className":48058,"style":35689},[274],[],{"type":150,"tag":169,"props":48061,"children":48063},{"className":48062,"style":35754},[348,349],[],{"type":150,"tag":169,"props":48065,"children":48067},{"className":48066},[355],[48068],{"type":155,"value":358},{"type":150,"tag":169,"props":48070,"children":48072},{"className":48071},[259],[48073],{"type":150,"tag":169,"props":48074,"children":48076},{"className":48075,"style":35768},[264],[48077],{"type":150,"tag":169,"props":48078,"children":48079},{},[],{"type":150,"tag":151,"props":48081,"children":48082},{},[48083],{"type":155,"value":48084},"于是原本弱者为了生存开发出来的 efficiency innovation，最后反而可能让强者 scale 得更狠。这才是最残酷的一层。所以问题最后又回到：省下来的钱，值不值得花。",{"type":150,"tag":372,"props":48086,"children":48088},{"id":48087},"小结",[48089],{"type":155,"value":48087},{"type":150,"tag":151,"props":48091,"children":48092},{},[48093],{"type":155,"value":48094},"开头说 ds4.1f 是模型架构与硬件 co-design 的杰作。看完整条路线，我觉得这个说法可以讲得更具体一点：它省掉的不是某一项。是账单上的好几项，而且是分几代攒下来的。把整条路线画成一张图是这样：",{"type":150,"tag":34778,"props":48096,"children":48098},{"className":34780,"code":48097,"language":155,"meta":28,"style":28},"MHA              每个 head 一份 K、V\n │\n ▼\nMQA \u002F GQA        压 head 数：离散地共享\n │\n ▼\nMLA              压 D 轴：连续压缩成 latent\n │\n ▼\nCSA \u002F HCA        压 N 轴：压成 block，再挑着看\n │\n ▼\nLinear \u002F SSM     压掉保存本身：只留一个 state\n │\n ▼\nV4.1             压 L 轴和计算阶段：不再每层每步都付同样的钱\n",[48099],{"type":150,"tag":34784,"props":48100,"children":48101},{"__ignoreMap":28},[48102,48110,48118,48126,48134,48141,48148,48156,48164,48172,48181,48189,48197,48206,48214,48222],{"type":150,"tag":169,"props":48103,"children":48104},{"class":34789,"line":34790},[48105],{"type":150,"tag":169,"props":48106,"children":48107},{},[48108],{"type":155,"value":48109},"MHA              每个 head 一份 K、V\n",{"type":150,"tag":169,"props":48111,"children":48112},{"class":34789,"line":34799},[48113],{"type":150,"tag":169,"props":48114,"children":48115},{},[48116],{"type":155,"value":48117}," │\n",{"type":150,"tag":169,"props":48119,"children":48120},{"class":34789,"line":34808},[48121],{"type":150,"tag":169,"props":48122,"children":48123},{},[48124],{"type":155,"value":48125}," ▼\n",{"type":150,"tag":169,"props":48127,"children":48128},{"class":34789,"line":34817},[48129],{"type":150,"tag":169,"props":48130,"children":48131},{},[48132],{"type":155,"value":48133},"MQA \u002F GQA        压 head 数：离散地共享\n",{"type":150,"tag":169,"props":48135,"children":48136},{"class":34789,"line":34825},[48137],{"type":150,"tag":169,"props":48138,"children":48139},{},[48140],{"type":155,"value":48117},{"type":150,"tag":169,"props":48142,"children":48143},{"class":34789,"line":34883},[48144],{"type":150,"tag":169,"props":48145,"children":48146},{},[48147],{"type":155,"value":48125},{"type":150,"tag":169,"props":48149,"children":48150},{"class":34789,"line":34891},[48151],{"type":150,"tag":169,"props":48152,"children":48153},{},[48154],{"type":155,"value":48155},"MLA              压 D 轴：连续压缩成 latent\n",{"type":150,"tag":169,"props":48157,"children":48159},{"class":34789,"line":48158},8,[48160],{"type":150,"tag":169,"props":48161,"children":48162},{},[48163],{"type":155,"value":48117},{"type":150,"tag":169,"props":48165,"children":48167},{"class":34789,"line":48166},9,[48168],{"type":150,"tag":169,"props":48169,"children":48170},{},[48171],{"type":155,"value":48125},{"type":150,"tag":169,"props":48173,"children":48175},{"class":34789,"line":48174},10,[48176],{"type":150,"tag":169,"props":48177,"children":48178},{},[48179],{"type":155,"value":48180},"CSA \u002F HCA        压 N 轴：压成 block，再挑着看\n",{"type":150,"tag":169,"props":48182,"children":48184},{"class":34789,"line":48183},11,[48185],{"type":150,"tag":169,"props":48186,"children":48187},{},[48188],{"type":155,"value":48117},{"type":150,"tag":169,"props":48190,"children":48192},{"class":34789,"line":48191},12,[48193],{"type":150,"tag":169,"props":48194,"children":48195},{},[48196],{"type":155,"value":48125},{"type":150,"tag":169,"props":48198,"children":48200},{"class":34789,"line":48199},13,[48201],{"type":150,"tag":169,"props":48202,"children":48203},{},[48204],{"type":155,"value":48205},"Linear \u002F SSM     压掉保存本身：只留一个 state\n",{"type":150,"tag":169,"props":48207,"children":48209},{"class":34789,"line":48208},14,[48210],{"type":150,"tag":169,"props":48211,"children":48212},{},[48213],{"type":155,"value":48117},{"type":150,"tag":169,"props":48215,"children":48217},{"class":34789,"line":48216},15,[48218],{"type":150,"tag":169,"props":48219,"children":48220},{},[48221],{"type":155,"value":48125},{"type":150,"tag":169,"props":48223,"children":48225},{"class":34789,"line":48224},16,[48226],{"type":150,"tag":169,"props":48227,"children":48228},{},[48229],{"type":155,"value":48230},"V4.1             压 L 轴和计算阶段：不再每层每步都付同样的钱\n",{"type":150,"tag":151,"props":48232,"children":48233},{},[48234],{"type":155,"value":48235},"同样的路线，换成一连串追问：",{"type":150,"tag":169,"props":48237,"children":48239},{"className":48238},[172],[48240],{"type":150,"tag":169,"props":48241,"children":48243},{"className":48242},[177],[48244,48282],{"type":150,"tag":169,"props":48245,"children":48247},{"className":48246},[182],[48248],{"type":150,"tag":185,"props":48249,"children":48250},{"xmlns":187,"display":188},[48251],{"type":150,"tag":191,"props":48252,"children":48253},{},[48254,48277],{"type":150,"tag":195,"props":48255,"children":48256},{},[48257,48262,48265,48269,48272],{"type":150,"tag":217,"props":48258,"children":48259},{},[48260],{"type":155,"value":48261},"每个 head 都需要独立 KV 吗？",{"type":150,"tag":558,"props":48263,"children":48264},{"width":3491},[],{"type":150,"tag":417,"props":48266,"children":48267},{},[48268],{"type":155,"value":12954},{"type":150,"tag":558,"props":48270,"children":48271},{"width":3491},[],{"type":150,"tag":217,"props":48273,"children":48274},{},[48275],{"type":155,"value":48276},"MQA \u002F GQA",{"type":150,"tag":223,"props":48278,"children":48279},{"encoding":225},[48280],{"type":155,"value":48281},"\\text{每个 head 都需要独立 KV 吗？} \\quad \\rightarrow \\quad \\text{MQA \u002F GQA}",{"type":150,"tag":169,"props":48283,"children":48285},{"className":48284,"ariaHidden":214},[232],[48286,48349],{"type":150,"tag":169,"props":48287,"children":48289},{"className":48288},[237],[48290,48294,48328,48332,48336,48341,48345],{"type":150,"tag":169,"props":48291,"children":48293},{"className":48292,"style":942},[242],[],{"type":150,"tag":169,"props":48295,"children":48297},{"className":48296},[248,155],[48298,48304,48310,48316,48322],{"type":150,"tag":169,"props":48299,"children":48301},{"className":48300},[248,297],[48302],{"type":155,"value":48303},"每个",{"type":150,"tag":169,"props":48305,"children":48307},{"className":48306},[248],[48308],{"type":155,"value":48309}," head ",{"type":150,"tag":169,"props":48311,"children":48313},{"className":48312},[248,297],[48314],{"type":155,"value":48315},"都需要独立",{"type":150,"tag":169,"props":48317,"children":48319},{"className":48318},[248],[48320],{"type":155,"value":48321}," KV ",{"type":150,"tag":169,"props":48323,"children":48325},{"className":48324},[248,297],[48326],{"type":155,"value":48327},"吗？",{"type":150,"tag":169,"props":48329,"children":48331},{"className":48330,"style":4087},[558],[],{"type":150,"tag":169,"props":48333,"children":48335},{"className":48334,"style":839},[558],[],{"type":150,"tag":169,"props":48337,"children":48339},{"className":48338},[844],[48340],{"type":155,"value":12954},{"type":150,"tag":169,"props":48342,"children":48344},{"className":48343,"style":4087},[558],[],{"type":150,"tag":169,"props":48346,"children":48348},{"className":48347,"style":839},[558],[],{"type":150,"tag":169,"props":48350,"children":48352},{"className":48351},[237],[48353,48357],{"type":150,"tag":169,"props":48354,"children":48356},{"className":48355,"style":999},[242],[],{"type":150,"tag":169,"props":48358,"children":48360},{"className":48359},[248,155],[48361],{"type":150,"tag":169,"props":48362,"children":48364},{"className":48363},[248],[48365],{"type":155,"value":48276},{"type":150,"tag":169,"props":48367,"children":48369},{"className":48368},[172],[48370],{"type":150,"tag":169,"props":48371,"children":48373},{"className":48372},[177],[48374,48411],{"type":150,"tag":169,"props":48375,"children":48377},{"className":48376},[182],[48378],{"type":150,"tag":185,"props":48379,"children":48380},{"xmlns":187,"display":188},[48381],{"type":150,"tag":191,"props":48382,"children":48383},{},[48384,48406],{"type":150,"tag":195,"props":48385,"children":48386},{},[48387,48392,48395,48399,48402],{"type":150,"tag":217,"props":48388,"children":48389},{},[48390],{"type":155,"value":48391},"每个 token 都需要完整 KV 吗？",{"type":150,"tag":558,"props":48393,"children":48394},{"width":3491},[],{"type":150,"tag":417,"props":48396,"children":48397},{},[48398],{"type":155,"value":12954},{"type":150,"tag":558,"props":48400,"children":48401},{"width":3491},[],{"type":150,"tag":217,"props":48403,"children":48404},{},[48405],{"type":155,"value":13},{"type":150,"tag":223,"props":48407,"children":48408},{"encoding":225},[48409],{"type":155,"value":48410},"\\text{每个 token 都需要完整 KV 吗？} \\quad \\rightarrow \\quad \\text{MLA}",{"type":150,"tag":169,"props":48412,"children":48414},{"className":48413,"ariaHidden":214},[232],[48415,48474],{"type":150,"tag":169,"props":48416,"children":48418},{"className":48417},[237],[48419,48423,48453,48457,48461,48466,48470],{"type":150,"tag":169,"props":48420,"children":48422},{"className":48421,"style":942},[242],[],{"type":150,"tag":169,"props":48424,"children":48426},{"className":48425},[248,155],[48427,48432,48437,48443,48448],{"type":150,"tag":169,"props":48428,"children":48430},{"className":48429},[248,297],[48431],{"type":155,"value":48303},{"type":150,"tag":169,"props":48433,"children":48435},{"className":48434},[248],[48436],{"type":155,"value":35541},{"type":150,"tag":169,"props":48438,"children":48440},{"className":48439},[248,297],[48441],{"type":155,"value":48442},"都需要完整",{"type":150,"tag":169,"props":48444,"children":48446},{"className":48445},[248],[48447],{"type":155,"value":48321},{"type":150,"tag":169,"props":48449,"children":48451},{"className":48450},[248,297],[48452],{"type":155,"value":48327},{"type":150,"tag":169,"props":48454,"children":48456},{"className":48455,"style":4087},[558],[],{"type":150,"tag":169,"props":48458,"children":48460},{"className":48459,"style":839},[558],[],{"type":150,"tag":169,"props":48462,"children":48464},{"className":48463},[844],[48465],{"type":155,"value":12954},{"type":150,"tag":169,"props":48467,"children":48469},{"className":48468,"style":4087},[558],[],{"type":150,"tag":169,"props":48471,"children":48473},{"className":48472,"style":839},[558],[],{"type":150,"tag":169,"props":48475,"children":48477},{"className":48476},[237],[48478,48482],{"type":150,"tag":169,"props":48479,"children":48481},{"className":48480,"style":5854},[242],[],{"type":150,"tag":169,"props":48483,"children":48485},{"className":48484},[248,155],[48486],{"type":150,"tag":169,"props":48487,"children":48489},{"className":48488},[248],[48490],{"type":155,"value":13},{"type":150,"tag":169,"props":48492,"children":48494},{"className":48493},[172],[48495],{"type":150,"tag":169,"props":48496,"children":48498},{"className":48497},[177],[48499,48536],{"type":150,"tag":169,"props":48500,"children":48502},{"className":48501},[182],[48503],{"type":150,"tag":185,"props":48504,"children":48505},{"xmlns":187,"display":188},[48506],{"type":150,"tag":191,"props":48507,"children":48508},{},[48509,48531],{"type":150,"tag":195,"props":48510,"children":48511},{},[48512,48517,48520,48524,48527],{"type":150,"tag":217,"props":48513,"children":48514},{},[48515],{"type":155,"value":48516},"每个 query 都需要看所有 token 吗？",{"type":150,"tag":558,"props":48518,"children":48519},{"width":3491},[],{"type":150,"tag":417,"props":48521,"children":48522},{},[48523],{"type":155,"value":12954},{"type":150,"tag":558,"props":48525,"children":48526},{"width":3491},[],{"type":150,"tag":217,"props":48528,"children":48529},{},[48530],{"type":155,"value":36048},{"type":150,"tag":223,"props":48532,"children":48533},{"encoding":225},[48534],{"type":155,"value":48535},"\\text{每个 query 都需要看所有 token 吗？} \\quad \\rightarrow \\quad \\text{CSA \u002F HCA}",{"type":150,"tag":169,"props":48537,"children":48539},{"className":48538,"ariaHidden":214},[232],[48540,48600],{"type":150,"tag":169,"props":48541,"children":48543},{"className":48542},[237],[48544,48548,48579,48583,48587,48592,48596],{"type":150,"tag":169,"props":48545,"children":48547},{"className":48546,"style":1121},[242],[],{"type":150,"tag":169,"props":48549,"children":48551},{"className":48550},[248,155],[48552,48557,48563,48569,48574],{"type":150,"tag":169,"props":48553,"children":48555},{"className":48554},[248,297],[48556],{"type":155,"value":48303},{"type":150,"tag":169,"props":48558,"children":48560},{"className":48559},[248],[48561],{"type":155,"value":48562}," query ",{"type":150,"tag":169,"props":48564,"children":48566},{"className":48565},[248,297],[48567],{"type":155,"value":48568},"都需要看所有",{"type":150,"tag":169,"props":48570,"children":48572},{"className":48571},[248],[48573],{"type":155,"value":35541},{"type":150,"tag":169,"props":48575,"children":48577},{"className":48576},[248,297],[48578],{"type":155,"value":48327},{"type":150,"tag":169,"props":48580,"children":48582},{"className":48581,"style":4087},[558],[],{"type":150,"tag":169,"props":48584,"children":48586},{"className":48585,"style":839},[558],[],{"type":150,"tag":169,"props":48588,"children":48590},{"className":48589},[844],[48591],{"type":155,"value":12954},{"type":150,"tag":169,"props":48593,"children":48595},{"className":48594,"style":4087},[558],[],{"type":150,"tag":169,"props":48597,"children":48599},{"className":48598,"style":839},[558],[],{"type":150,"tag":169,"props":48601,"children":48603},{"className":48602},[237],[48604,48608],{"type":150,"tag":169,"props":48605,"children":48607},{"className":48606,"style":999},[242],[],{"type":150,"tag":169,"props":48609,"children":48611},{"className":48610},[248,155],[48612],{"type":150,"tag":169,"props":48613,"children":48615},{"className":48614},[248],[48616],{"type":155,"value":36048},{"type":150,"tag":169,"props":48618,"children":48620},{"className":48619},[172],[48621],{"type":150,"tag":169,"props":48622,"children":48624},{"className":48623},[177],[48625,48663],{"type":150,"tag":169,"props":48626,"children":48628},{"className":48627},[182],[48629],{"type":150,"tag":185,"props":48630,"children":48631},{"xmlns":187,"display":188},[48632],{"type":150,"tag":191,"props":48633,"children":48634},{},[48635,48658],{"type":150,"tag":195,"props":48636,"children":48637},{},[48638,48643,48646,48650,48653],{"type":150,"tag":217,"props":48639,"children":48640},{},[48641],{"type":155,"value":48642},"每个 token 都需要被永久保存吗？",{"type":150,"tag":558,"props":48644,"children":48645},{"width":3491},[],{"type":150,"tag":417,"props":48647,"children":48648},{},[48649],{"type":155,"value":12954},{"type":150,"tag":558,"props":48651,"children":48652},{"width":3491},[],{"type":150,"tag":217,"props":48654,"children":48655},{},[48656],{"type":155,"value":48657},"Linear \u002F SSM",{"type":150,"tag":223,"props":48659,"children":48660},{"encoding":225},[48661],{"type":155,"value":48662},"\\text{每个 token 都需要被永久保存吗？} \\quad \\rightarrow \\quad \\text{Linear \u002F SSM}",{"type":150,"tag":169,"props":48664,"children":48666},{"className":48665,"ariaHidden":214},[232],[48667,48716],{"type":150,"tag":169,"props":48668,"children":48670},{"className":48669},[237],[48671,48675,48695,48699,48703,48708,48712],{"type":150,"tag":169,"props":48672,"children":48674},{"className":48673,"style":942},[242],[],{"type":150,"tag":169,"props":48676,"children":48678},{"className":48677},[248,155],[48679,48684,48689],{"type":150,"tag":169,"props":48680,"children":48682},{"className":48681},[248,297],[48683],{"type":155,"value":48303},{"type":150,"tag":169,"props":48685,"children":48687},{"className":48686},[248],[48688],{"type":155,"value":35541},{"type":150,"tag":169,"props":48690,"children":48692},{"className":48691},[248,297],[48693],{"type":155,"value":48694},"都需要被永久保存吗？",{"type":150,"tag":169,"props":48696,"children":48698},{"className":48697,"style":4087},[558],[],{"type":150,"tag":169,"props":48700,"children":48702},{"className":48701,"style":839},[558],[],{"type":150,"tag":169,"props":48704,"children":48706},{"className":48705},[844],[48707],{"type":155,"value":12954},{"type":150,"tag":169,"props":48709,"children":48711},{"className":48710,"style":4087},[558],[],{"type":150,"tag":169,"props":48713,"children":48715},{"className":48714,"style":839},[558],[],{"type":150,"tag":169,"props":48717,"children":48719},{"className":48718},[237],[48720,48724],{"type":150,"tag":169,"props":48721,"children":48723},{"className":48722,"style":999},[242],[],{"type":150,"tag":169,"props":48725,"children":48727},{"className":48726},[248,155],[48728],{"type":150,"tag":169,"props":48729,"children":48731},{"className":48730},[248],[48732],{"type":155,"value":48657},{"type":150,"tag":169,"props":48734,"children":48736},{"className":48735},[172],[48737],{"type":150,"tag":169,"props":48738,"children":48740},{"className":48739},[177],[48741,48778],{"type":150,"tag":169,"props":48742,"children":48744},{"className":48743},[182],[48745],{"type":150,"tag":185,"props":48746,"children":48747},{"xmlns":187,"display":188},[48748],{"type":150,"tag":191,"props":48749,"children":48750},{},[48751,48773],{"type":150,"tag":195,"props":48752,"children":48753},{},[48754,48759,48762,48766,48769],{"type":150,"tag":217,"props":48755,"children":48756},{},[48757],{"type":155,"value":48758},"每个 layer 都需要独立 memory 吗？",{"type":150,"tag":558,"props":48760,"children":48761},{"width":3491},[],{"type":150,"tag":417,"props":48763,"children":48764},{},[48765],{"type":155,"value":12954},{"type":150,"tag":558,"props":48767,"children":48768},{"width":3491},[],{"type":150,"tag":217,"props":48770,"children":48771},{},[48772],{"type":155,"value":36126},{"type":150,"tag":223,"props":48774,"children":48775},{"encoding":225},[48776],{"type":155,"value":48777},"\\text{每个 layer 都需要独立 memory 吗？} \\quad \\rightarrow \\quad \\text{V4.1}",{"type":150,"tag":169,"props":48779,"children":48781},{"className":48780,"ariaHidden":214},[232],[48782,48841],{"type":150,"tag":169,"props":48783,"children":48785},{"className":48784},[237],[48786,48790,48820,48824,48828,48833,48837],{"type":150,"tag":169,"props":48787,"children":48789},{"className":48788,"style":1121},[242],[],{"type":150,"tag":169,"props":48791,"children":48793},{"className":48792},[248,155],[48794,48799,48805,48810,48815],{"type":150,"tag":169,"props":48795,"children":48797},{"className":48796},[248,297],[48798],{"type":155,"value":48303},{"type":150,"tag":169,"props":48800,"children":48802},{"className":48801},[248],[48803],{"type":155,"value":48804}," layer ",{"type":150,"tag":169,"props":48806,"children":48808},{"className":48807},[248,297],[48809],{"type":155,"value":48315},{"type":150,"tag":169,"props":48811,"children":48813},{"className":48812},[248],[48814],{"type":155,"value":318},{"type":150,"tag":169,"props":48816,"children":48818},{"className":48817},[248,297],[48819],{"type":155,"value":48327},{"type":150,"tag":169,"props":48821,"children":48823},{"className":48822,"style":4087},[558],[],{"type":150,"tag":169,"props":48825,"children":48827},{"className":48826,"style":839},[558],[],{"type":150,"tag":169,"props":48829,"children":48831},{"className":48830},[844],[48832],{"type":155,"value":12954},{"type":150,"tag":169,"props":48834,"children":48836},{"className":48835,"style":4087},[558],[],{"type":150,"tag":169,"props":48838,"children":48840},{"className":48839,"style":839},[558],[],{"type":150,"tag":169,"props":48842,"children":48844},{"className":48843},[237],[48845,48849],{"type":150,"tag":169,"props":48846,"children":48848},{"className":48847,"style":5854},[242],[],{"type":150,"tag":169,"props":48850,"children":48852},{"className":48851},[248,155],[48853],{"type":150,"tag":169,"props":48854,"children":48856},{"className":48855},[248],[48857],{"type":155,"value":36126},{"type":150,"tag":169,"props":48859,"children":48861},{"className":48860},[172],[48862],{"type":150,"tag":169,"props":48863,"children":48865},{"className":48864},[177],[48866,48903],{"type":150,"tag":169,"props":48867,"children":48869},{"className":48868},[182],[48870],{"type":150,"tag":185,"props":48871,"children":48872},{"xmlns":187,"display":188},[48873],{"type":150,"tag":191,"props":48874,"children":48875},{},[48876,48898],{"type":150,"tag":195,"props":48877,"children":48878},{},[48879,48884,48887,48891,48894],{"type":150,"tag":217,"props":48880,"children":48881},{},[48882],{"type":155,"value":48883},"Prefill 和 Decode 必须用同样计算吗？",{"type":150,"tag":558,"props":48885,"children":48886},{"width":3491},[],{"type":150,"tag":417,"props":48888,"children":48889},{},[48890],{"type":155,"value":12954},{"type":150,"tag":558,"props":48892,"children":48893},{"width":3491},[],{"type":150,"tag":217,"props":48895,"children":48896},{},[48897],{"type":155,"value":36126},{"type":150,"tag":223,"props":48899,"children":48900},{"encoding":225},[48901],{"type":155,"value":48902},"\\text{Prefill 和 Decode 必须用同样计算吗？} \\quad \\rightarrow \\quad \\text{V4.1}",{"type":150,"tag":169,"props":48904,"children":48906},{"className":48905,"ariaHidden":214},[232],[48907,48963],{"type":150,"tag":169,"props":48908,"children":48910},{"className":48909},[237],[48911,48915,48942,48946,48950,48955,48959],{"type":150,"tag":169,"props":48912,"children":48914},{"className":48913,"style":942},[242],[],{"type":150,"tag":169,"props":48916,"children":48918},{"className":48917},[248,155],[48919,48925,48930,48936],{"type":150,"tag":169,"props":48920,"children":48922},{"className":48921},[248],[48923],{"type":155,"value":48924},"Prefill ",{"type":150,"tag":169,"props":48926,"children":48928},{"className":48927},[248,297],[48929],{"type":155,"value":324},{"type":150,"tag":169,"props":48931,"children":48933},{"className":48932},[248],[48934],{"type":155,"value":48935}," Decode ",{"type":150,"tag":169,"props":48937,"children":48939},{"className":48938},[248,297],[48940],{"type":155,"value":48941},"必须用同样计算吗？",{"type":150,"tag":169,"props":48943,"children":48945},{"className":48944,"style":4087},[558],[],{"type":150,"tag":169,"props":48947,"children":48949},{"className":48948,"style":839},[558],[],{"type":150,"tag":169,"props":48951,"children":48953},{"className":48952},[844],[48954],{"type":155,"value":12954},{"type":150,"tag":169,"props":48956,"children":48958},{"className":48957,"style":4087},[558],[],{"type":150,"tag":169,"props":48960,"children":48962},{"className":48961,"style":839},[558],[],{"type":150,"tag":169,"props":48964,"children":48966},{"className":48965},[237],[48967,48971],{"type":150,"tag":169,"props":48968,"children":48970},{"className":48969,"style":5854},[242],[],{"type":150,"tag":169,"props":48972,"children":48974},{"className":48973},[248,155],[48975],{"type":150,"tag":169,"props":48976,"children":48978},{"className":48977},[248],[48979],{"type":155,"value":36126},{"type":150,"tag":151,"props":48981,"children":48982},{},[48983],{"type":155,"value":48984},"每一代都在放松一个假设。而这三个动作一直循环：",{"type":150,"tag":169,"props":48986,"children":48988},{"className":48987},[172],[48989],{"type":150,"tag":169,"props":48990,"children":48992},{"className":48991},[177],[48993,49050],{"type":150,"tag":169,"props":48994,"children":48996},{"className":48995},[182],[48997],{"type":150,"tag":185,"props":48998,"children":48999},{"xmlns":187,"display":188},[49000],{"type":150,"tag":191,"props":49001,"children":49002},{},[49003,49045],{"type":150,"tag":195,"props":49004,"children":49005},{},[49006],{"type":150,"tag":199,"props":49007,"children":49008},{"notation":201},[49009],{"type":150,"tag":204,"props":49010,"children":49011},{"scriptlevel":206,"displaystyle":207},[49012],{"type":150,"tag":204,"props":49013,"children":49014},{"scriptlevel":206,"displaystyle":207},[49015],{"type":150,"tag":204,"props":49016,"children":49017},{"scriptlevel":206,"displaystyle":214},[49018],{"type":150,"tag":195,"props":49019,"children":49020},{},[49021,49026,49031,49036,49040],{"type":150,"tag":217,"props":49022,"children":49023},{},[49024],{"type":155,"value":49025},"保存信息",{"type":150,"tag":417,"props":49027,"children":49028},{},[49029],{"type":155,"value":49030},"↔",{"type":150,"tag":217,"props":49032,"children":49033},{},[49034],{"type":155,"value":49035},"压缩信息",{"type":150,"tag":417,"props":49037,"children":49038},{},[49039],{"type":155,"value":49030},{"type":150,"tag":217,"props":49041,"children":49042},{},[49043],{"type":155,"value":49044},"找回信息",{"type":150,"tag":223,"props":49046,"children":49047},{"encoding":225},[49048],{"type":155,"value":49049},"\\boxed{\\text{保存信息} \\leftrightarrow \\text{压缩信息} \\leftrightarrow \\text{找回信息}}",{"type":150,"tag":169,"props":49051,"children":49053},{"className":49052,"ariaHidden":214},[232],[49054],{"type":150,"tag":169,"props":49055,"children":49057},{"className":49056},[237],[49058,49062],{"type":150,"tag":169,"props":49059,"children":49061},{"className":49060,"style":36221},[242],[],{"type":150,"tag":169,"props":49063,"children":49065},{"className":49064},[248],[49066],{"type":150,"tag":169,"props":49067,"children":49069},{"className":49068},[253,254],[49070,49166],{"type":150,"tag":169,"props":49071,"children":49073},{"className":49072},[259],[49074,49161],{"type":150,"tag":169,"props":49075,"children":49077},{"className":49076,"style":265},[264],[49078,49150],{"type":150,"tag":169,"props":49079,"children":49080},{"style":36241},[49081,49085],{"type":150,"tag":169,"props":49082,"children":49084},{"className":49083,"style":36246},[274],[],{"type":150,"tag":169,"props":49086,"children":49088},{"className":49087},[280],[49089],{"type":150,"tag":169,"props":49090,"children":49092},{"className":49091},[248],[49093],{"type":150,"tag":169,"props":49094,"children":49096},{"className":49095},[248],[49097,49106,49110,49115,49119,49128,49132,49137,49141],{"type":150,"tag":169,"props":49098,"children":49100},{"className":49099},[248,155],[49101],{"type":150,"tag":169,"props":49102,"children":49104},{"className":49103},[248,297],[49105],{"type":155,"value":49025},{"type":150,"tag":169,"props":49107,"children":49109},{"className":49108,"style":839},[558],[],{"type":150,"tag":169,"props":49111,"children":49113},{"className":49112},[844],[49114],{"type":155,"value":49030},{"type":150,"tag":169,"props":49116,"children":49118},{"className":49117,"style":839},[558],[],{"type":150,"tag":169,"props":49120,"children":49122},{"className":49121},[248,155],[49123],{"type":150,"tag":169,"props":49124,"children":49126},{"className":49125},[248,297],[49127],{"type":155,"value":49035},{"type":150,"tag":169,"props":49129,"children":49131},{"className":49130,"style":839},[558],[],{"type":150,"tag":169,"props":49133,"children":49135},{"className":49134},[844],[49136],{"type":155,"value":49030},{"type":150,"tag":169,"props":49138,"children":49140},{"className":49139,"style":839},[558],[],{"type":150,"tag":169,"props":49142,"children":49144},{"className":49143},[248,155],[49145],{"type":150,"tag":169,"props":49146,"children":49148},{"className":49147},[248,297],[49149],{"type":155,"value":49044},{"type":150,"tag":169,"props":49151,"children":49152},{"style":339},[49153,49157],{"type":150,"tag":169,"props":49154,"children":49156},{"className":49155,"style":36246},[274],[],{"type":150,"tag":169,"props":49158,"children":49160},{"className":49159,"style":36279},[348,349],[],{"type":150,"tag":169,"props":49162,"children":49164},{"className":49163},[355],[49165],{"type":155,"value":358},{"type":150,"tag":169,"props":49167,"children":49169},{"className":49168},[259],[49170],{"type":150,"tag":169,"props":49171,"children":49173},{"className":49172,"style":35056},[264],[49174],{"type":150,"tag":169,"props":49175,"children":49176},{},[],{"type":150,"tag":151,"props":49178,"children":49179},{},[49180],{"type":155,"value":49181},"最后那句话还可以再收一下。",{"type":150,"tag":151,"props":49183,"children":49184},{},[49185],{"type":155,"value":49186},"前面一直把架构、数据、算力当成三样不同的东西比轻重，这个切法可能一开始就偏了。它们互相兑换：",{"type":150,"tag":151,"props":49188,"children":49189},{},[49190,49195],{"type":150,"tag":158,"props":49191,"children":49192},{},[49193],{"type":155,"value":49194},"数据是架构。",{"type":155,"value":49196}," 用什么数据、怎么组织、怎么采样，直接决定模型长成什么样。同一套 attention，换一个数据分布就是另一个模型。数据管线里每个决定，效果上和改架构是同一个量级。",{"type":150,"tag":151,"props":49198,"children":49199},{},[49200,49205],{"type":150,"tag":158,"props":49201,"children":49202},{},[49203],{"type":155,"value":49204},"架构是算力。",{"type":155,"value":49206}," 这条前面已经推过一遍。MLA 把 KV 压到 1\u002F57，ds4.1f 把 decode 的算术强度压到离拐点 300 倍，省下来的都是算力。架构不改模型会做什么，只改做同一件事要花多少钱。它是算力的另一种记账方式。",{"type":150,"tag":151,"props":49208,"children":49209},{},[49210,49215],{"type":150,"tag":158,"props":49211,"children":49212},{},[49213],{"type":155,"value":49214},"算力是数据。",{"type":155,"value":49216}," 省下来的算力不会消失，它会被重新花到更多 tokens、更长的 RL rollout、更多 environment interaction、更多 verifier compute。算力的下游就是数据。",{"type":150,"tag":151,"props":49218,"children":49219},{},[49220],{"type":155,"value":49221},"三个词绕成一个环。所以能力和架构谁更重要，这个问题本身可能就问错了：它们不是三条并行的赛道，而是同一笔资源的三种形态。换哪个名字都不改变总量，变的只是兑换率。",{"type":150,"tag":151,"props":49223,"children":49224},{},[49225],{"type":155,"value":49226},"那当成本压到足够低之后，架构这条线上的竞争还剩多少意义？我的答案是，它的意义从来不是自己创造能力，而是决定那笔资源能兑换多少次。",{"type":150,"tag":49228,"props":49229,"children":49230},"style",{},[49231],{"type":155,"value":49232},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":28,"searchDepth":34799,"depth":34808,"links":49234},[49235,49236,49237,49238,49239,49243,49244,49247,49248,49249,49250],{"id":374,"depth":34799,"text":377},{"id":6208,"depth":34799,"text":6211},{"id":7823,"depth":34799,"text":7826},{"id":10009,"depth":34799,"text":13},{"id":30750,"depth":34799,"text":30753,"children":49240},[49241,49242],{"id":30757,"depth":34808,"text":30760},{"id":31980,"depth":34808,"text":31983},{"id":32376,"depth":34799,"text":32379},{"id":34630,"depth":34799,"text":34633,"children":49245},[49246],{"id":34768,"depth":34808,"text":34771},{"id":35582,"depth":34799,"text":35585},{"id":44713,"depth":34799,"text":44716},{"id":46715,"depth":34799,"text":46715},{"id":48087,"depth":34799,"text":48087},"markdown","content:posts:llm-memory-compute-tradeoff.md","content","posts\u002Fllm-memory-compute-tradeoff.md","posts\u002Fllm-memory-compute-tradeoff","md",1789088134184]