{"id":123506,"date":"2026-11-07T15:25:04","date_gmt":"2026-11-07T07:25:04","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/123506.html"},"modified":"2026-11-07T15:25:04","modified_gmt":"2026-11-07T07:25:04","slug":"%e7%ac%ac3%e7%ab%a0-spark4-x-%e9%83%a8%e7%bd%b2%e6%a8%a1%e5%bc%8f%e3%80%81%e8%b5%84%e6%ba%90%e6%a8%a1%e5%9e%8b%e4%b8%8e%e5%ad%98%e9%87%8f%e8%bf%81%e7%a7%bb","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/123506.html","title":{"rendered":"\u7b2c3\u7ae0 Spark4.x \u90e8\u7f72\u6a21\u5f0f\u3001\u8d44\u6e90\u6a21\u578b\u4e0e\u5b58\u91cf\u8fc1\u79fb"},"content":{"rendered":"<p>        \/* MD \/ \u5bcc\u6587\u672c\u4e2d\u7684 .toc\uff08\u542b\u535a\u5ba2\u56ed\u642c\u5bb6\u7b49\u5d4c\u5957\u7ed3\u6784\uff09\uff1b.toc-box \u5728\u4fa7\u680f\uff0c\u4e0d\u53d7\u5f71\u54cd *\/<br \/>\n        #content_views .toc,<br \/>\n        \/* \u7f16\u8f91\u5668\u5e38\u5728\u76ee\u5f55\u524d\u540e\u63d2\u5165\u7a7a p\uff08:empty \u4ecd\u5360 20px\uff09\uff0c\u4e00\u5e76\u53bb\u6389\u907f\u514d\u9876\u7a7a\u9699 *\/<br \/>\n        #content_views.markdown_views &gt; p:empty:has(+ .toc),<br \/>\n        #content_views.markdown_views &gt; .toc + p:empty,<br \/>\n        \/* \u5bcc\u6587\u672c\u65e7\u7248\u76ee\u5f55\u6807\u8bb0 *\/<br \/>\n        #content_views.htmledit_views #main-toc,<br \/>\n        #content_views.htmledit_views #hr-toc,<br \/>\n        #content_views.htmledit_views p[id*=\\&#8221;-toc\\&#8221;] {<br \/>\n          display: none !important;<br \/>\n        }<br \/>\n        \/* \u76ee\u5f55\u53bb\u6389\u540e\uff0c\u7d27\u8ddf\u7684\u9996\u4e2a\u6807\u9898\u4e0d\u518d\u591a\u51fa\u4e00\u5757\u4e0a\u8fb9\u8ddd *\/<br \/>\n        #content_views.markdown_views &gt; .toc + h1,<br \/>\n        #content_views.markdown_views &gt; .toc + h2,<br \/>\n        #content_views.markdown_views &gt; .toc + h3,<br \/>\n        #content_views.markdown_views &gt; .toc + h4,<br \/>\n        #content_views.markdown_views &gt; .toc + p:empty + h1,<br \/>\n        #content_views.markdown_views &gt; .toc + p:empty + h2,<br \/>\n        #content_views.markdown_views &gt; .toc + p:empty + h3,<br \/>\n        #content_views.markdown_views &gt; .toc + p:empty + h4 {<br \/>\n          margin-top: 0 !important;<br \/>\n        }<\/p>\n<h2>\u7b2c3\u7ae0 Spark4.x \u90e8\u7f72\u6a21\u5f0f\u3001\u8d44\u6e90\u6a21\u578b\u4e0e\u5b58\u91cf\u8fc1\u79fb<\/h2>\n<p>\u7ae0\u8282\u5b9a\u4f4d&#xff1a;\u4e0a\u4e00\u7ae0\u8ba9 Spark \u80fd\u8dd1&#xff0c;\u8fd9\u4e00\u7ae0\u8ba9\u5b83\u5728\u4e00\u4e2a\u5df2\u7ecf\u6709 Flink \u7684\u96c6\u7fa4\u91cc\u8dd1\u5f97\u4e0d\u51fa\u4e8b\u3002\u672c\u7ae0\u4f1a\u649e\u89c1\u672c\u7cfb\u5217\u7b2c\u4e00\u4e2a\u771f\u6b63\u8ba9\u4eba\u71ac\u591c\u7684\u95ee\u9898\u2014\u2014\u5bb9\u5668\u4e3a\u4ec0\u4e48\u4f1a\u88ab\u6740\u6389\u3002\u7b54\u6848\u4e0d\u5728 Spark \u7684\u65e5\u5fd7\u91cc&#xff0c;\u5728 YARN \/ Kubernetes \u7684\u4e8b\u4ef6\u91cc&#xff0c;\u8fd9\u4e5f\u662f\u5f88\u591a\u4eba\u67e5\u4e86\u4e09\u5c0f\u65f6\u8fd8\u56de\u5934\u6539 spark.executor.memory \u7684\u539f\u56e0&#xff1a;\u6539\u9519\u4e86\u5730\u65b9\u3002<\/p>\n<p>\u53e6\u4e00\u6761\u4e3b\u7ebf\u662f\u5171\u5b58\u3002\u628a Spark \u585e\u8fdb\u5df2\u6709 Flink \u7684\u751f\u4ea7\u96c6\u7fa4&#xff0c;\u8d44\u6e90\u4e89\u62a2\u662f\u660e\u9762\u4e0a\u7684\u95ee\u9898&#xff0c;\u5199\u5165\u6743\u51b2\u7a81\u662f\u6697\u5904\u7684\u95ee\u9898&#xff0c;\u800c\u540e\u8005\u4f1a\u9020\u6210\u6570\u636e\u4e22\u5931\u800c\u4e0d\u662f\u4f5c\u4e1a\u5931\u8d25\u3002<\/p>\n<hr \/>\n<h3>\u5b66\u4e60\u76ee\u6807<\/h3>\n<ul>\n<li>\u2713 \u80fd\u6309\u56e2\u961f\u73b0\u72b6\u9009\u51fa local \/ standalone \/ YARN \/ Kubernetes \u56db\u79cd\u6a21\u5f0f\u4e4b\u4e00&#xff0c;\u5e76\u8bf4\u6e05 client \u4e0e cluster \u90e8\u7f72\u6a21\u5f0f\u7684\u5dee\u522b\u5230\u5e95\u5f71\u54cd\u4ec0\u4e48<\/li>\n<li>\u2713 \u62ff\u5230 spark.executor.memory&#061;8g&#xff0c;\u80fd\u7b97\u51fa\u5bb9\u5668\u5b9e\u9645\u5411\u96c6\u7fa4\u7533\u8bf7\u4e86\u591a\u5c11\u3001\u5176\u4e2d\u591a\u5c11\u80fd\u7ed9\u8ba1\u7b97\u7528<\/li>\n<li>\u2713 \u8bf4\u6e05\u7edf\u4e00\u5185\u5b58\u533a M \u4e0e\u4e0d\u53ef\u9a71\u9010\u533a R \u7684\u5173\u7cfb&#xff0c;\u4ee5\u53ca\u4e3a\u4ec0\u4e48&#034;\u7f13\u5b58\u6324\u6389\u6267\u884c&#034;\u8fd9\u4ef6\u4e8b\u5728 Spark \u91cc\u65b9\u5411\u662f\u5355\u5411\u7684<\/li>\n<li>\u2713 \u9047\u5230&#034;\u5bb9\u5668\u88ab\u6740&#034;\u65f6&#xff0c;\u80fd\u6309\u56fa\u5b9a\u8def\u5f84\u5728 10 \u5206\u949f\u5185\u5b9a\u4f4d\u5230\u662f overhead \u4e0d\u591f\u3001off-heap \u6ca1\u5f00\u3001\u8fd8\u662f GC \u628a\u5806\u6491\u6ee1<\/li>\n<li>\u2713 \u89e3\u91ca\u52a8\u6001\u5206\u533a\u5206\u914d\u7684\u4e09\u4e2a\u6613\u9519\u70b9&#xff0c;\u4ee5\u53ca\u5b83\u548c AQE \u5408\u5e76\u5206\u533a\u4e3a\u4ec0\u4e48\u4f1a\u4e92\u76f8\u62b5\u6d88<\/li>\n<li>\u2713 \u7ed9\u51fa Spark \u4e0e Flink \u5171\u96c6\u7fa4\u65f6\u7684\u8d44\u6e90\u9694\u79bb\u65b9\u6848&#xff0c;\u4ee5\u53ca\u540c\u4e00\u5f20\u6e56\u8868\u7684\u5199\u5165\u6743\u4ef2\u88c1\u673a\u5236<\/li>\n<li>\u2713 \u624b\u4e0a\u6709\u4e00\u4efd 15 \u9879\u7684\u4e0a\u7ebf\u8bc4\u5ba1\u6e05\u5355&#xff0c;\u53ef\u4ee5\u62ff\u53bb\u5f00\u4f1a<\/li>\n<\/ul>\n<hr \/>\n<h3>3.1 \u56db\u79cd\u90e8\u7f72\u6a21\u5f0f\u4e0e\u9009\u578b<\/h3>\n<h4>\u8868 3-1 \u90e8\u7f72\u6a21\u5f0f\u5bf9\u7167<\/h4>\n<table>\n<tr>\u6a21\u5f0f\u96c6\u7fa4\u7ba1\u7406\u5668\u9002\u7528\u4e0d\u9002\u7528\u672c\u7cfb\u5217\u7528\u6cd5<\/tr>\n<tbody>\n<tr>\n<td>local<\/td>\n<td>\u65e0&#xff08;\u540c\u8fdb\u7a0b\u591a\u7ebf\u7a0b&#xff09;<\/td>\n<td>\u5355\u673a\u5f00\u53d1\u3001\u5355\u5143\u9a8c\u8bc1\u3001\u672c\u7ae0 2.4 \u7684\u95ed\u73af<\/td>\n<td>\u4efb\u4f55\u9700\u8981\u6a2a\u5411\u6269\u5c55\u7684\u573a\u666f&#xff1b;local[*] \u5728\u5bb9\u5668\u91cc\u4f1a\u8bfb\u5bbf\u4e3b\u673a\u6838\u6570<\/td>\n<td>\u7b2c 2 \u7ae0\u5168\u90e8\u5b9e\u9a8c<\/td>\n<\/tr>\n<tr>\n<td>standalone<\/td>\n<td>Spark \u81ea\u5e26 Master\/Worker<\/td>\n<td>\u6559\u5b66\u3001\u5c0f\u89c4\u6a21\u72ec\u7acb\u96c6\u7fa4\u3001\u4e0d\u60f3\u5f15\u5165 Hadoop<\/td>\n<td>\u5df2\u6709 YARN\/K8s \u7684\u56e2\u961f&#xff1b;\u591a\u79df\u6237\u9694\u79bb\u5f31&#xff1b;\u751f\u6001\u5de5\u5177\u7f3a<\/td>\n<td>\u4e0d\u91c7\u7528<\/td>\n<\/tr>\n<tr>\n<td>YARN<\/td>\n<td>Hadoop YARN<\/td>\n<td>\u5df2\u6709 Hadoop \u751f\u6001\u3001\u9700\u8981\u961f\u5217\u4e0e\u5bb9\u91cf\u8c03\u5ea6\u3001\u6279\u4f5c\u4e1a\u4e0e Hive\/MapReduce \u5171\u5b58<\/td>\n<td>\u5f39\u6027\u4f38\u7f29\u7c92\u5ea6\u7c97&#xff1b;\u5bb9\u5668\u5316\u4ea4\u4ed8\u4e0d\u53cb\u597d<\/td>\n<td>\u4e3b\u7ebf\u4e4b\u4e00&#xff08;\u4e0e\u65e2\u6709 Hadoop \u5171\u5b58&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>Kubernetes<\/td>\n<td>K8s<\/td>\n<td>\u4e91\u539f\u751f\u6808\u3001\u4e0e Flink on K8s \u5171\u6c60\u3001\u9700\u8981\u955c\u50cf\u5316\u4ea4\u4ed8\u4e0e\u7ec6\u5f39\u6027<\/td>\n<td>\u9700\u8981\u81ea\u5df1\u7ba1 driver \u7684\u66b4\u9732\u4e0e\u9274\u6743&#xff1b;native \u5ba2\u6237\u7aef\u8981\u6c42\u955c\u50cf\u91cc\u6709 Spark<\/td>\n<td>\u4e3b\u7ebf\u4e4b\u4e8c&#xff08;\u4e0e Flink \u5171\u6c60&#xff09;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u9009\u578b\u53ea\u6709\u4e00\u6761\u5224\u65ad\u4f9d\u636e&#xff1a;\u4f60\u7684 Flink \u8dd1\u5728\u54ea&#xff0c;Spark \u5c31\u8ddf\u7740\u8dd1\u5728\u54ea\u3002 \u4e24\u5957\u5f15\u64ce\u5206\u5c5e\u4e0d\u540c\u8d44\u6e90\u6c60&#xff0c;\u7b2c 3.4 \u8282\u6240\u6709\u7684\u9694\u79bb\u624b\u6bb5\u90fd\u4f1a\u5931\u6548\u2014\u2014\u56e0\u4e3a\u4f60\u65e0\u6cd5\u5728\u4e24\u4e2a\u4e92\u4e0d\u76f8\u5e72\u7684\u8c03\u5ea6\u5668\u4e4b\u95f4\u505a\u4ef2\u88c1\u3002<\/p>\n<h4>3.1.1 client \u4e0e cluster&#xff1a;\u4e0d\u662f&#034;\u6027\u80fd\u5dee\u5f02&#034;&#xff0c;\u662f&#034;driver \u5728\u54ea&#034;<\/h4>\n<p>\u8fd9\u662f YARN\/K8s \u4e0a\u6700\u5bb9\u6613\u8bef\u89e3\u7684\u4e00\u7ec4\u6982\u5ff5\u3002\u5dee\u522b\u53ea\u6709\u4e00\u4e2a&#xff1a;Driver \u8fdb\u7a0b\u8dd1\u5728\u54ea\u91cc\u3002<\/p>\n<p>client \u6a21\u5f0f                          cluster \u6a21\u5f0f<br \/>\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500            \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500<br \/>\n\u63d0\u4ea4\u673a                               \u96c6\u7fa4\u5185\u4e00\u4e2a\u5bb9\u5668<br \/>\n \u2514\u2500 Driver&#xff08;\u4f60\u7684\u7ec8\u7aef&#xff09;                \u2514\u2500 Driver&#xff08;ApplicationMaster \/ driver pod&#xff09;<br \/>\n      \u2191                                   \u2191<br \/>\n      \u2502  \u7f51\u7edc\u5fc5\u987b\u80fd\u76f4\u8fde\u6bcf\u4e2a Executor        \u2502  \u53ea\u9700\u63d0\u4ea4\u673a\u8fde\u5230 ResourceManager \/ API Server<br \/>\n      \u2502  &#xff08;\u56de\u8fde driver \u7684 4040 \u7aef\u53e3&#xff09;       \u2502<br \/>\n \u63d0\u4ea4\u673a\u4e0b\u7ebf \u2192 \u4f5c\u4e1a\u5931\u8054                 \u63d0\u4ea4\u673a\u4e0b\u7ebf \u2192 \u4f5c\u4e1a\u7ee7\u7eed<\/p>\n<p>\u9002\u7528&#xff1a;\u4ea4\u4e92\u5f0f\u3001spark-shell\u3001Notebook    \u9002\u7528&#xff1a;\u6240\u6709\u5b9a\u65f6\u6279\u4f5c\u4e1a\u3001\u751f\u4ea7\u8c03\u5ea6<\/p>\n<p>\u4e3a\u4ec0\u4e48\u751f\u4ea7\u6279\u4f5c\u4e1a\u5fc5\u987b cluster \u6a21\u5f0f&#xff1a;\u8c03\u5ea6\u5668&#xff08;DolphinScheduler \/ CronJob&#xff09;\u7684\u6267\u884c\u8282\u70b9\u662f\u77ed\u751f\u547d\u5468\u671f\u7684&#xff0c;driver \u6302\u5728\u5b83\u4e0a\u9762\u7b49\u4e8e\u7ed9\u4f5c\u4e1a\u52a0\u4e86\u4e00\u4e2a&#034;\u63d0\u4ea4\u673a\u4e0d\u80fd\u6389\u7ebf&#034;\u7684\u9690\u542b\u4f9d\u8d56\u3002\u8fd9\u6761\u5728\u6392\u73ed\u8868\u4e0a\u8868\u73b0\u4e3a&#034;\u4f5c\u4e1a\u6bcf\u5929\u51cc\u6668 2 \u70b9\u968f\u673a\u5931\u8d25&#xff0c;\u4e14\u5931\u8d25\u539f\u56e0\u6bcf\u6b21\u90fd\u4e0d\u4e00\u6837&#034;\u3002<\/p>\n<p>K8s \u4e0a\u591a\u4e00\u91cd\u9ebb\u70e6&#xff1a;client \u6a21\u5f0f\u4e0b driver \u5728\u96c6\u7fa4\u5916&#xff0c;executor pod \u8981\u56de\u8fde\u5b83&#xff0c;\u9700\u8981\u6253\u901a\u7f51\u7edc\u4e0e\u9274\u6743\u3002\u5b98\u65b9\u6587\u6863\u660e\u786e\u5efa\u8bae\u751f\u4ea7\u7528 cluster \u6a21\u5f0f&#xff0c;\u672c\u7cfb\u5217\u4e0d\u63d0\u4f9b K8s client \u6a21\u5f0f\u7684\u914d\u7f6e\u3002<\/p>\n<h4>3.1.2 YARN \u90e8\u7f72\u9aa8\u67b6<\/h4>\n<p><span class=\"token comment\"># conf\/spark-defaults.conf \u8ffd\u52a0<\/span><br \/>\nspark.master                     <span class=\"token function\">yarn<\/span><br \/>\nspark.deploy.mode                cluster<br \/>\nspark.yarn.submit.waitAppCompletion  <span class=\"token boolean\">false<\/span>   <span class=\"token comment\"># \u8c03\u5ea6\u5668\u573a\u666f&#xff1a;\u63d0\u4ea4\u5373\u8fd4\u56de&#xff0c;\u72b6\u6001\u4ea4\u7ed9\u8c03\u5ea6\u5668\u8f6e\u8be2<\/span><\/p>\n<p><span class=\"token comment\"># \u961f\u5217\u4e0e\u4f5c\u4e1a\u6807\u8bc6&#xff08;\u751f\u4ea7\u5fc5\u586b&#xff0c;\u7b2c 3.4 \u8282\u7684\u9694\u79bb\u9760\u5b83&#xff09;<\/span><br \/>\nspark.yarn.queue                 batch_normal<br \/>\nspark.yarn.tags                  <span class=\"token assign-left variable\">owner<\/span><span class=\"token operator\">&#061;<\/span>yangjun,team<span class=\"token operator\">&#061;<\/span>dt,sla<span class=\"token operator\">&#061;<\/span>T1<\/p>\n<p><span class=\"token comment\"># AM&#xff08;driver \u5bb9\u5668&#xff09;\u8d44\u6e90<\/span><br \/>\nspark.driver.memory              4g<br \/>\nspark.driver.cores               <span class=\"token number\">2<\/span><br \/>\nspark.executor.memory            8g<br \/>\nspark.executor.cores             <span class=\"token number\">4<\/span><br \/>\nspark.executor.memoryOverhead    2g<\/p>\n<p>\u53c2\u6570\u8bf4\u660e<\/p>\n<table>\n<tr>\u53c2\u6570\u4e3a\u4ec0\u4e48\u8fd9\u4e48\u8bbe<\/tr>\n<tbody>\n<tr>\n<td>spark.yarn.submit.waitAppCompletion&#061;false<\/td>\n<td>\u9ed8\u8ba4 true \u4f1a\u963b\u585e\u5230\u4f5c\u4e1a\u7ed3\u675f\u3002\u8c03\u5ea6\u5668\u573a\u666f\u4e0b\u8fd9\u4f1a\u5360\u4f4f\u4e00\u4e2a\u8c03\u5ea6\u7ebf\u7a0b\u6570\u5c0f\u65f6&#xff0c;\u662f&#034;\u8c03\u5ea6\u5668\u88ab\u6279\u4f5c\u4e1a\u62d6\u6b7b&#034;\u7684\u7ecf\u5178\u6210\u56e0<\/td>\n<\/tr>\n<tr>\n<td>spark.yarn.tags<\/td>\n<td>\u961f\u5217\u76d1\u63a7\u4e0e\u6210\u672c\u5f52\u96c6\u7684\u552f\u4e00\u6293\u624b\u3002\u4e0d\u5e26 tag \u7684\u4f5c\u4e1a\u5728\u7b2c 17 \u7ae0\u6210\u672c\u6cbb\u7406\u91cc\u662f&#034;\u65e0\u6cd5\u5f52\u56e0&#034;\u7684\u652f\u51fa<\/td>\n<\/tr>\n<tr>\n<td>spark.executor.cores&#061;4<\/td>\n<td>\u4e0d\u662f\u8d8a\u5927\u8d8a\u597d\u3002\u6838\u6570\u589e\u52a0\u4f1a\u8ba9\u540c executor \u5185\u5e76\u53d1 task \u53d8\u591a&#xff0c;\u6bcf\u4e2a task \u5206\u5230\u7684\u5185\u5b58\u540c\u6bd4\u51cf\u5c11&#xff0c;\u5bb9\u6613\u4ece&#034;\u591f\u7528&#034;\u53d8\u6210&#034;spill \u5230\u78c1\u76d8&#034;\u30023.2 \u7ed9\u7b97\u6cd5<\/td>\n<\/tr>\n<tr>\n<td>spark.executor.memoryOverhead&#061;2g<\/td>\n<td>8g \u00d7 0.10 &#061; 0.8g \u662f\u9ed8\u8ba4\u503c&#xff0c;\u4f46\u672c\u4f5c\u4e1a\u8981\u8bfb Parquet&#xff08;\u5411\u91cf\u5316\u8bfb\u53d6\u8d70\u5806\u5916&#xff09;\u4e14\u5e26 Python UDF&#xff0c;\u9ed8\u8ba4\u503c\u5fc5\u7136\u4e0d\u591f\u3002\u8fd9\u662f 3.2 \u7684\u6838\u5fc3\u6848\u4f8b<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;spark-submit &#8211;version \u540e\u63d0\u4ea4\u4e00\u4e2a &#8211;class org.apache.spark.examples.SparkPi&#xff0c;\u5728 YARN ResourceManager UI \u770b\u5230\u5e94\u7528\u72b6\u6001 ACCEPTED \u2192 RUNNING \u2192 FINISHED&#xff0c;\u4e14 AM Node \u4e0e Container \u6570\u91cf\u7b26\u5408\u9884\u671f\u3002<\/p>\n<h4>3.1.3 Kubernetes \u90e8\u7f72\u9aa8\u67b6<\/h4>\n<p>spark-submit <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">&#8211;master<\/span> k8s:\/\/https:\/\/k8s-api:6443 <span class=\"token punctuation\">\\\\<\/span><br \/>\n  &#8211;deploy-mode cluster <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">&#8211;conf<\/span> <span class=\"token assign-left variable\">spark.kubernetes.container.image<\/span><span class=\"token operator\">&#061;<\/span>registry.internal\/spark:4.1.4 <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">&#8211;conf<\/span> <span class=\"token assign-left variable\">spark.kubernetes.namespace<\/span><span class=\"token operator\">&#061;<\/span>data-batch <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">&#8211;conf<\/span> <span class=\"token assign-left variable\">spark.kubernetes.authenticate.driver.serviceAccountName<\/span><span class=\"token operator\">&#061;<\/span>spark-batch-sa <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">&#8211;conf<\/span> <span class=\"token assign-left variable\">spark.kubernetes.executor.deleteOnTermination<\/span><span class=\"token operator\">&#061;<\/span>true <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">&#8211;conf<\/span> <span class=\"token assign-left variable\">spark.executor.memory<\/span><span class=\"token operator\">&#061;<\/span>8g <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">&#8211;conf<\/span> <span class=\"token assign-left variable\">spark.executor.memoryOverhead<\/span><span class=\"token operator\">&#061;<\/span>2g <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">&#8211;conf<\/span> <span class=\"token assign-left variable\">spark.kubernetes.driver.request.cores<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token number\">2<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">&#8211;conf<\/span> <span class=\"token assign-left variable\">spark.kubernetes.executor.request.cores<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token number\">1<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">&#8211;conf<\/span> <span class=\"token assign-left variable\">spark.kubernetes.executor.limit.cores<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token number\">4<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  local:\/\/\/opt\/spark\/examples\/jars\/spark-examples_2.13-4.1.4.jar<\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff08;\u53ea\u8bb2\u4e0e YARN \u8bed\u4e49\u4e0d\u540c\u7684&#xff09;<\/p>\n<table>\n<tr>\u53c2\u6570\u8bf4\u660e<\/tr>\n<tbody>\n<tr>\n<td>executor.request.cores vs limit.cores<\/td>\n<td>K8s \u7684 request\/limit \u5206\u79bb\u8bed\u4e49\u3002request&#061;1 \/ limit&#061;4 \u8ba9 executor \u5728\u7a7a\u95f2\u65f6\u53efburst&#xff0c;\u662f\u4e0e Flink \u5171\u6c60\u65f6\u7684\u5173\u952e\u65cb\u94ae&#xff08;3.4&#xff09;\u3002YARN \u6ca1\u6709\u8fd9\u4e2a\u6982\u5ff5<\/td>\n<\/tr>\n<tr>\n<td>executor.deleteOnTermination&#061;true<\/td>\n<td>\u9ed8\u8ba4\u884c\u4e3a\u4e0b\u5931\u8d25 executor pod \u4f1a\u4fdd\u7559\u7528\u4e8e\u6392\u67e5\u3002\u751f\u4ea7\u5fc5\u987b\u5f00&#xff0c;\u5426\u5219\u4e00\u6b21\u503e\u659c\u4e8b\u6545\u7559\u4e0b\u4e0a\u5343\u4e2a Completed pod&#xff0c;\u628a etcd \u538b\u57ae<\/td>\n<\/tr>\n<tr>\n<td>authenticate.driver.serviceAccountName<\/td>\n<td>driver pod \u9700\u8981\u56de\u8fde API Server \u521b\u5efa executor pod&#xff0c;\u7f3a\u6743\u9650\u7684\u62a5\u9519\u5f62\u6001\u662f forbidden: cannot create service<\/td>\n<\/tr>\n<tr>\n<td>\u955c\u50cf tag 4.1.4<\/td>\n<td>\u955c\u50cf\u7248\u672c\u5fc5\u987b\u4e0e &#8211;class\/jar \u7684 Spark \u7248\u672c\u4e00\u81f4\u3002\u6df7\u7528\u7684\u62a5\u9519\u5728 driver \u542f\u52a8\u9636\u6bb5&#xff0c;\u5f62\u6001\u662f NoSuchMethodError&#xff0c;\u5f88\u5bb9\u6613\u88ab\u8bef\u5224\u6210 connector \u95ee\u9898<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;kubectl get pods -n data-batch -w \u770b\u5230 xxx-driver \u5148 Running&#xff0c;\u968f\u540e executor pod \u9010\u4e2a\u51fa\u73b0&#xff1b;\u4f5c\u4e1a\u7ed3\u675f\u540e driver \u8fdb\u5165 Completed&#xff0c;executor \u88ab\u6e05\u7406\u3002<\/p>\n<h4>3.1.4 native \u5ba2\u6237\u7aef&#xff1a;\u4e00\u4e2a\u4f1a\u8ba9\u4eba\u56f0\u60d1\u7684\u5f62\u6001<\/h4>\n<p>Spark 4 \u8d77\u5b98\u65b9\u63d0\u4f9b Spark Connect \u4f5c\u4e3a\u9ed8\u8ba4\u5165\u53e3&#xff0c;spark-submit \u7684\u884c\u4e3a\u4e5f\u968f\u4e4b\u53d8\u5316&#xff1a;\u5b83\u4f1a\u5148\u8d77\u4e00\u4e2a Connect server \u518d\u63d0\u4ea4\u3002\u5728 K8s \u4e0a\u8fd9\u5e26\u6765\u4e24\u79cd\u955c\u50cf\u5f62\u6001&#xff1a;<\/p>\n<ul>\n<li>\u666e\u901a\u955c\u50cf&#xff1a;spark-submit \u8d70\u4f20\u7edf\u8def\u5f84&#xff0c;driver pod \u5185\u542b Spark \u8fd0\u884c\u65f6<\/li>\n<li>Connect \u670d\u52a1\u5316\u90e8\u7f72&#xff1a;\u5e38\u9a7b Connect server&#xff0c;\u5ba2\u6237\u7aef\u53ea\u9700 pyspark \u5305&#xff0c;\u4e0d\u9700\u8981\u672c\u5730\u6709 Spark \u53d1\u884c\u7248<\/li>\n<\/ul>\n<p>\u7b2c 16 \u7ae0\u5c55\u5f00\u670d\u52a1\u5316\u65f6\u518d\u7ec6\u8bb2\u3002\u672c\u7ae0\u53ea\u9700\u8981\u8bb0\u4f4f\u4e00\u6761\u6392\u969c\u4e8b\u5b9e&#xff1a;\u5982\u679c\u5ba2\u6237\u7aef\u4e0e server \u7684 Spark \u5c0f\u7248\u672c\u4e0d\u4e00\u81f4&#xff0c;\u62a5\u9519\u53d1\u751f\u5728 plan \u5e8f\u5217\u5316\u9636\u6bb5&#xff0c;\u9519\u8bef\u4fe1\u606f\u91cc\u4e0d\u4f1a\u51fa\u73b0\u7248\u672c\u5b57\u6837&#xff0c;\u800c\u662f Unrecognized \/ unsupported \u7c7b\u63d0\u793a\u3002\u8fd9\u662f Spark Connect \u65f6\u4ee3\u65b0\u589e\u7684\u4e00\u7c7b&#034;\u770b\u8d77\u6765\u50cf bug \u5176\u5b9e\u662f\u7248\u672c\u9519\u914d&#034;\u3002<\/p>\n<hr \/>\n<h3>3.2 \u5185\u5b58\u6a21\u578b&#xff1a;\u5bb9\u5668\u88ab\u6740\u7684\u771f\u6b63\u539f\u56e0<\/h3>\n<h4>3.2.1 \u5148\u5efa\u7acb\u6b63\u786e\u7684 mental model<\/h4>\n<p>\u7edd\u5927\u591a\u6570\u4eba\u4ee5\u4e3a spark.executor.memory \u5c31\u662f executor \u5360\u7528\u7684\u5185\u5b58\u3002\u4e0d\u662f\u3002 \u5b83\u53ea\u662f JVM \u5806\u5927\u5c0f\u3002<\/p>\n<p>   \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 \u5bb9\u5668\u603b\u5185\u5b58&#xff08;\u96c6\u7fa4\u7ba1\u7406\u5668\u770b\u5230\u7684&#xff09;\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n   \u2502                                                              \u2502<br \/>\n   \u2502  \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 JVM \u5806 &#061; spark.executor.memory&#xff08;\u5982 8g&#xff09;\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510  \u2502<br \/>\n   \u2502  \u2502                                                        \u2502  \u2502<br \/>\n   \u2502  \u2502   300 MiB \u9884\u7559&#xff08;Spark \u5185\u90e8\u5143\u6570\u636e&#xff0c;\u5b98\u65b9\u660e\u786e\u6263\u9664&#xff09;         \u2502  \u2502<br \/>\n   \u2502  \u2502  \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 \u7edf\u4e00\u533a M &#061; (heap \u2212 300MiB) \u00d7 0.6 \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510  \u2502  \u2502<br \/>\n   \u2502  \u2502  \u2502  \u6267\u884c\u5185\u5b58&#xff08;shuffle \/ join \/ sort \/ aggregate&#xff09;     \u2502  \u2502  \u2502<br \/>\n   \u2502  \u2502  \u2502  \u250c\u2500\u2500\u2500\u2500 \u4e0d\u53ef\u9a71\u9010\u533a R &#061; M \u00d7 0.5 \u2500\u2500\u2500\u2500\u2510               \u2502  \u2502  \u2502<br \/>\n   \u2502  \u2502  \u2502  \u2502  storage&#xff1a;\u7f13\u5b58\u7684 block          \u2502  \u2190 \u6267\u884c\u4e5f\u4e0d\u9a71\u9010\u5b83 \u2502  \u2502  \u2502<br \/>\n   \u2502  \u2502  \u2502  \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518               \u2502  \u2502  \u2502<br \/>\n   \u2502  \u2502  \u2502  M \u5185\u5176\u4f59&#xff1a;storage \u4e0e execution \u4e92\u76f8\u501f\u7528            \u2502  \u2502  \u2502<br \/>\n   \u2502  \u2502  \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518  \u2502  \u2502<br \/>\n   \u2502  \u2502   \u5176\u4f59 40%&#xff1a;\u7528\u6237\u6570\u636e\u7ed3\u6784\u3001\u5bf9\u8c61\u5934\u3001Spark \u5185\u90e8\u5143\u6570\u636e\u3001       \u2502  \u2502<br \/>\n   \u2502  \u2502            \u7a00\u758f\/\u8d85\u5927\u8bb0\u5f55\u7684 OOM \u7f13\u51b2                      \u2502  \u2502<br \/>\n   \u2502  \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518  \u2502<br \/>\n\u2502  \u2502                                                              \u2502<br \/>\n   \u2502  &#043; \u5806\u5916 spark.memory.offHeap.size&#xff08;\u9ed8\u8ba4 0&#xff0c;\u987b\u663e\u5f0f\u5f00\u542f&#xff09;        \u2502<br \/>\n   \u2502  &#043; spark.executor.memoryOverhead&#xff08;\u9ed8\u8ba4 heap\u00d70.10&#xff0c;\u4e0b\u9650 384m&#xff09;  \u2502<br \/>\n   \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<\/p>\n<p>\u56fe 3-1 Executor \u5185\u5b58\u5206\u5c42<\/p>\n<p>\u56db\u4e2a\u6570\u5b57\u5fc5\u987b\u8bb0\u4f4f&#xff1a;<\/p>\n<table>\n<tr>\u9879\u9ed8\u8ba4\u542b\u4e49<\/tr>\n<tbody>\n<tr>\n<td>spark.memory.fraction<\/td>\n<td>0.6<\/td>\n<td>M \u5360 (\u5806 \u2212 300MiB) \u7684\u6bd4\u4f8b\u3002\u5b98\u65b9\u539f\u6587&#xff1a;\u201cexpresses the size of M as a fraction of the (JVM heap space &#8211; 300MiB)\u201d<\/td>\n<\/tr>\n<tr>\n<td>spark.memory.storageFraction<\/td>\n<td>0.5<\/td>\n<td>R \u5360 M \u7684\u6bd4\u4f8b\u3002R \u5185\u7684\u7f13\u5b58\u5757\u6c38\u4e0d\u88ab\u6267\u884c\u9a71\u9010<\/td>\n<\/tr>\n<tr>\n<td>spark.executor.memoryOverhead<\/td>\n<td>heap \u00d7 0.10&#xff0c;\u4e0b\u9650 384 MiB<\/td>\n<td>\u5806\u5916 &#043; \u975e\u5806\u7684\u5bb9\u5668\u4f59\u91cf<\/td>\n<\/tr>\n<tr>\n<td>300 MiB<\/td>\n<td>\u56fa\u5b9a\u6263\u9664<\/td>\n<td>Spark \u5185\u90e8\u5143\u6570\u636e\u9884\u7559&#xff0c;\u4e0d\u53c2\u4e0e M \u7684\u8ba1\u7b97<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>3.2.2 \u7edf\u4e00\u5185\u5b58\u533a\u7684\u5173\u952e\u6027\u8d28&#xff1a;\u9a71\u9010\u662f\u5355\u5411\u7684<\/h4>\n<p>\u5b98\u65b9\u63cf\u8ff0&#xff08;\u539f\u6587\u8981\u70b9&#xff09;&#xff1a;<\/p>\n<ul>\n<li>\u6267\u884c\u4e0e\u5b58\u50a8\u5171\u4eab\u4e00\u4e2a\u7edf\u4e00\u533a M\u3002\u4e00\u65b9\u4e0d\u7528\u65f6&#xff0c;\u53e6\u4e00\u65b9\u53ef\u5360\u6ee1\u5168\u90e8<\/li>\n<li>\u6267\u884c\u53ef\u4ee5\u9a71\u9010\u5b58\u50a8&#xff0c;\u4f46\u53ea\u80fd\u9a71\u9010\u5230\u603b\u91cf\u8dcc\u7834 R \u4e3a\u6b62<\/li>\n<li>\u5b58\u50a8\u4e0d\u80fd\u9a71\u9010\u6267\u884c\u2014\u2014\u5b98\u65b9\u7406\u7531\u662f&#034;\u5b9e\u73b0\u4e0a\u7684\u590d\u6742\u5ea6&#034;<\/li>\n<\/ul>\n<p>\u8fd9\u6761\u4e0d\u5bf9\u79f0\u6027\u76f4\u63a5\u51b3\u5b9a\u4e86\u4e24\u7c7b\u751f\u4ea7\u73b0\u8c61&#xff1a;<\/p>\n<p>\u73b0\u8c61 A&#xff1a;\u52a0\u4e86 cache() \u53cd\u800c\u53d8\u6162&#xff0c;\u4e14\u78c1\u76d8 spill \u66b4\u589e<br \/>\n  \u6839\u56e0&#xff1a;\u7f13\u5b58\u5757\u5360\u4f4f M&#xff0c;\u6267\u884c\u5185\u5b58\u88ab\u538b\u5230 R \u4e4b\u5916&#xff0c;\u89e6\u53d1 spill<br \/>\n  \u5224\u636e&#xff1a;Spark UI Storage \u9875\u6709\u7f13\u5b58&#xff0c;\u4f46 Shuffle\/Sort \u9636\u6bb5\u51fa\u73b0 spill<br \/>\n  \u5904\u7f6e&#xff1a;\u964d\u4f4e memory.fraction&#xff08;\u5b98\u65b9\u5efa\u8bae&#xff1a;&#034;cache fewer objects than slow down task execution&#034;&#xff09;<br \/>\n        \u6216\u53bb\u6389\u4e0d\u5fc5\u8981\u7684 cache()<\/p>\n<p>\u73b0\u8c61 B&#xff1a;\u7f13\u5b58\u88ab\u53cd\u590d\u9a71\u9010\u3001\u91cd\u7b97&#xff0c;CPU \u6da8\u4f46 IO \u6ca1\u6da8<br \/>\n  \u6839\u56e0&#xff1a;\u6267\u884c\u5185\u5b58\u9700\u6c42\u5927&#xff0c;\u628a R \u4e4b\u5916\u7684\u7f13\u5b58\u5757\u5168\u9a71\u9010\u4e86&#xff0c;\u4e0b\u6b21\u8bbf\u95ee\u8981\u91cd\u7b97<br \/>\n  \u5224\u636e&#xff1a;Storage \u9875\u67d0 RDD \u7684 &#034;Cached Size&#034; \u8fdc\u5c0f\u4e8e &#034;Size in Memory&#034;<br \/>\n  \u5904\u7f6e&#xff1a;\u63d0\u9ad8 memory.storageFraction&#xff0c;\u6216\u6269\u5927 executor \u5185\u5b58<\/p>\n<p>\u4e3a\u4ec0\u4e48\u65b9\u5411\u91cd\u8981&#xff1a;\u5982\u679c\u4ee5\u4e3a\u662f\u5bf9\u79f0\u7684&#xff0c;\u9047\u5230\u73b0\u8c61 A \u4f1a\u53bb\u8c03\u5927 memory.fraction&#xff08;\u60f3\u8ba9\u7f13\u5b58\u66f4\u5bbd\u677e&#xff09;&#xff0c;\u7ed3\u679c\u6267\u884c\u5185\u5b58\u88ab\u538b\u7f29\u3001spill \u66f4\u591a\u3001\u4f5c\u4e1a\u66f4\u6162\u3002\u8c03\u53cd\u65b9\u5411\u662f\u8fd9\u7c7b\u95ee\u9898\u6700\u5e38\u89c1\u7684\u8017\u65f6\u6765\u6e90\u3002<\/p>\n<h4>3.2.3 \u4e09\u79cd&#034;\u5bb9\u5668\u88ab\u6740&#034;\u7684\u533a\u5206\u4e0e\u5b9a\u4f4d<\/h4>\n<p>\u8fd9\u662f\u672c\u7ae0\u6700\u5b9e\u7528\u7684\u4e00\u8282\u3002\u4e09\u8005\u8868\u8c61\u76f8\u540c&#xff08;executor \u6d88\u5931\u3001\u4f5c\u4e1a\u5931\u8d25\u6216\u91cd\u8dd1&#xff09;&#xff0c;\u6210\u56e0\u4e0e\u89e3\u6cd5\u5b8c\u5168\u4e0d\u540c\u3002<\/p>\n<p>\u7b2c\u4e00\u6b65&#xff1a;\u5148\u770b\u662f\u8c01\u62a5\u7684\u9519<br \/>\n   YARN \u65e5\u5fd7\u542b &#034;Container killed by YARN for exceeding memory limits&#034;<br \/>\n   K8s \u4e8b\u4ef6\u542b &#034;OOMKilled&#034; \u6216 exit code 137<br \/>\n   Spark \u81ea\u8eab\u65e5\u5fd7\u542b &#034;java.lang.OutOfMemoryError: Java heap space&#034;<br \/>\n                                    \u2502<br \/>\n        \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n        \u25bc                           \u25bc                           \u25bc<br \/>\n   \u2460 \u88ab\u96c6\u7fa4\u7ba1\u7406\u5668\u6740            \u2461 \u88ab\u7ba1\u7406\u5668\u6740                 \u2462 \u88ab JVM \u81ea\u5df1\u6740<br \/>\n   \u8d85\u51fa\u5bb9\u5668\u9650\u989d                \u8d85\u51fa\u5bb9\u5668\u9650\u989d                 \u5806\u5185\u4e0d\u8db3<br \/>\n   \u539f\u56e0\u5728 overhead            \u539f\u56e0\u5728 off-heap             \u539f\u56e0\u5728\u5806\u5185<br \/>\n   \u89e3\u6cd5&#xff1a;\u52a0 memoryOverhead     \u89e3\u6cd5&#xff1a;\u5f00 off-heap \u6216\u52a0 overhead  \u89e3\u6cd5&#xff1a;\u52a0\u5806 \/ \u51cf task \u5e76\u53d1<\/p>\n<p>\u2460 overhead \u4e0d\u8db3&#xff08;\u6700\u9ad8\u9891&#xff09;<\/p>\n<p>\u5178\u578b\u89e6\u53d1\u6761\u4ef6&#xff1a;\u8bfb Parquet \u7528\u5411\u91cf\u5316\u8bfb\u53d6\u5668\u3001\u7528\u4e86 Python UDF\u3001\u5927\u91cf native \u5e93\u3001\u6216\u5355\u6761\u8bb0\u5f55\u7279\u522b\u5927&#xff08;\u8f66\u8054\u7f51\u91cc\u5e26\u5d4c\u5957\u4fe1\u53f7\u6570\u7ec4\u7684\u6d88\u606f\u5c31\u662f\u8fd9\u79cd&#xff09;\u3002<\/p>\n<p><span class=\"token comment\"># \u73b0\u8c61<\/span><br \/>\nWARN YarnScheduler: Lost executor <span class=\"token number\">3<\/span> on node5: Container on node5 <span class=\"token punctuation\">..<\/span>.<br \/>\n  launched <span class=\"token punctuation\">..<\/span>. failed <span class=\"token number\">137<\/span> <span class=\"token punctuation\">(<\/span>Exit code <span class=\"token number\">137<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token punctuation\">..<\/span>.<br \/>\n  Container killed by YARN <span class=\"token keyword\">for<\/span> exceeding memory limits.<br \/>\n  <span class=\"token number\">9.0<\/span> GB of <span class=\"token number\">9.0<\/span> GB physical memory used.<\/p>\n<p><span class=\"token comment\"># \u7b97\u8d26&#xff1a;heap 8g &#043; \u9ed8\u8ba4 overhead 0.8g &#061; \u5bb9\u5668 8.8g&#xff0c;\u4f46\u5b9e\u9645\u7528\u4e86 9.0g \u2192 \u6740<\/span><br \/>\n<span class=\"token comment\"># \u89e3\u6cd5&#xff08;\u4e09\u9009\u4e00&#xff0c;\u6309\u4f18\u5148\u7ea7&#xff09;<\/span><br \/>\nspark.executor.memoryOverhead    2g        <span class=\"token comment\"># \u76f4\u63a5\u52a0\u4f59\u91cf&#xff0c;\u6700\u7b80\u5355<\/span><br \/>\nspark.memory.offHeap.enabled   <span class=\"token boolean\">true<\/span>        <span class=\"token comment\"># \u628a native \u5f00\u9500\u632a\u5230\u53ef\u63a7\u7684\u5806\u5916\u914d\u989d<\/span><br \/>\nspark.sql.parquet.columnarReaderBatchSize <span class=\"token number\">2048<\/span>   <span class=\"token comment\"># \u51cf\u5c0f\u5411\u91cf\u5316\u8bfb\u53d6\u6279&#xff0c;\u964d\u5cf0\u503c<\/span><\/p>\n<p>\u2461 off-heap \u672a\u5f00<\/p>\n<p>Spark \u9ed8\u8ba4 \u4e0d\u542f\u7528 \u5806\u5916\u5185\u5b58&#xff08;spark.memory.offHeap.enabled&#061;false&#xff0c;size \u4e3a 0&#xff09;\u3002\u5b98\u65b9\u6587\u6863\u5efa\u8bae&#xff1a;\u6267\u884c\u5185\u5b58\u8f83\u5927\u3001\u4e14\u5e0c\u671b\u51cf\u5c11 GC \u538b\u529b\u65f6\u542f\u7528&#xff0c;\u7528 UnifiedMemoryManager \u7ba1\u7406\u5806\u5916\u3002<\/p>\n<p><span class=\"token comment\"># \u5f00\u542f\u540e\u5fc5\u987b\u540c\u6b65\u8c03\u4f4e\u5806&#xff0c;\u5426\u5219\u5bb9\u5668\u603b\u989d\u4e0d\u53d8\u4f46\u5806\u5916\u6324\u5360 overhead<\/span><br \/>\nspark.memory.offHeap.enabled   <span class=\"token boolean\">true<\/span><br \/>\nspark.memory.offHeap.size      4g<br \/>\nspark.executor.memory          6g        <span class=\"token comment\"># \u4ece 8g \u964d\u4e0b\u6765<\/span><br \/>\nspark.executor.memoryOverhead  2g<br \/>\n<span class=\"token comment\"># \u5bb9\u5668\u603b\u989d &#061; 6 &#043; 4 &#043; 2 &#061; 12g&#xff08;\u6bd4\u539f\u6765 8.8g \u5927&#xff0c;\u5fc5\u987b\u540c\u6b65\u8c03\u961f\u5217\u914d\u989d&#xff09;<\/span><\/p>\n<p>\u4e00\u6761\u5fc5\u987b\u5199\u8fdb\u8bc4\u5ba1\u6e05\u5355\u7684\u7eaa\u5f8b&#xff1a;\u5f00 off-heap \u65f6&#xff0c;\u5bb9\u5668\u603b\u989d\u662f\u4e09\u9879\u76f8\u52a0&#xff0c;\u4e0d\u662f\u5806\u5927\u5c0f\u3002\u53ea\u6539 offHeap.size \u4e0d\u6539 executor.memory&#xff0c;\u7b49\u4e8e\u628a\u5bb9\u5668\u9700\u6c42\u6084\u6084\u62ac\u9ad8&#xff0c;\u7136\u540e\u88ab\u7ba1\u7406\u5668\u6740\u6389\u2014\u2014\u800c\u4e14\u62a5\u9519\u4fe1\u606f\u91cc\u4e0d\u4f1a\u63d0 off-heap \u4e00\u4e2a\u5b57\u3002<\/p>\n<p>\u2462 JVM \u5806\u5185 OOM<\/p>\n<p>\u4e0e\u524d\u4e24\u8005\u672c\u8d28\u4e0d\u540c&#xff1a;\u8fd9\u662f Java \u5c42\u7684\u5f02\u5e38&#xff0c;Spark \u65e5\u5fd7\u91cc\u6709\u5b8c\u6574\u6808\u3002<\/p>\n<p>java.lang.OutOfMemoryError: Java heap space<br \/>\n    at org.apache.spark.util.collection.OpenHashSet&#8230;<\/p>\n<p>\u5224\u636e\u4e0e\u89e3\u6cd5&#xff1a;<\/p>\n<table>\n<tr>\u5224\u636e\u7ed3\u8bba<\/tr>\n<tbody>\n<tr>\n<td>\u6808\u9876\u5728 MapStatus \/ BlockManager<\/td>\n<td>driver \u5806\u4e0d\u591f&#xff08;task \u592a\u591a&#xff0c;MapStatus \u6c47\u603b\u6491\u7206 driver&#xff09;\u2192 \u52a0 spark.driver.memory&#xff0c;\u51cf shuffle.partitions<\/td>\n<\/tr>\n<tr>\n<td>\u6808\u9876\u5728\u805a\u5408\/\u8fde\u63a5\u7b97\u5b50<\/td>\n<td>\u5355 task \u6570\u636e\u91cf\u8fc7\u5927 \u2192 \u503e\u659c&#xff08;\u7b2c 17 \u7ae0&#xff09;\u6216 shuffle.partitions \u592a\u5c0f<\/td>\n<\/tr>\n<tr>\n<td>GC \u65e5\u5fd7\u663e\u793a full GC \u5728 task \u5b8c\u6210\u524d\u591a\u6b21\u89e6\u53d1<\/td>\n<td>\u5b98\u65b9\u5224\u636e&#xff1a;\u201cit means that there isn\u2019t enough memory available for executing tasks\u201d \u2192 \u52a0\u5806\u6216\u51cf\u5e76\u53d1<\/td>\n<\/tr>\n<tr>\n<td>OldGen \u63a5\u8fd1\u6ee1<\/td>\n<td>\u5b98\u65b9\u5efa\u8bae\u964d\u4f4e spark.memory.fraction&#xff08;\u5b81\u53ef\u5c11\u7f13\u5b58&#xff09;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>3.2.4 JDK 17 \u4e0e G1GC&#xff1a;4.x \u7684\u4e00\u4e2a\u9690\u6027\u53d8\u66f4<\/h4>\n<p>\u5b98\u65b9\u539f\u6587&#xff08;Tuning \u9875&#xff09;&#xff1a;<\/p>\n<p>Since 4.0.0, Spark uses JDK 17 by default, which also makes the G1GC garbage collector the default. Note that with large executor heap sizes, it may be important to increase the G1 region size with -XX:G1HeapRegionSize.<\/p>\n<p>\u8fd9\u6761\u5bf9\u8f66\u8054\u7f51\u573a\u666f\u6709\u5b9e\u9645\u610f\u4e49&#xff1a;G1 \u7684 region \u5927\u5c0f\u9ed8\u8ba4\u6309\u5806\u5bb9\u91cf\u7684 1\/2048 \u63a8\u7b97&#xff0c;\u5806\u8d8a\u5927 region \u8d8a\u5927&#xff0c;\u4f46 humongous \u5bf9\u8c61\u7684\u5224\u5b9a\u9608\u503c\u662f region \u7684\u4e00\u534a\u3002\u5355\u6761\u9065\u6d4b\u6d88\u606f\u5e26\u5927\u6570\u7ec4\u65f6&#xff0c;\u4e00\u4e2a record batch \u5c31\u53ef\u80fd\u88ab\u5224\u5b9a\u4e3a humongous&#xff0c;\u76f4\u63a5\u8fdb\u5165 OldGen \u5e76\u89e6\u53d1\u989d\u5916\u5e76\u53d1\u6807\u8bb0\u5468\u671f\u3002<\/p>\n<p><span class=\"token comment\"># \u5927\u5806 executor&#xff08;\u2265 16g&#xff09;\u5efa\u8bae\u663e\u5f0f\u8bbe region size<\/span><br \/>\nspark.executor.memory        16g<br \/>\nspark.executor.extraJavaOptions  <span class=\"token parameter variable\">-XX:&#043;UseG1GC<\/span> <span class=\"token parameter variable\">-XX:G1HeapRegionSize<\/span><span class=\"token operator\">&#061;<\/span>16m -Xlog:gc*:file<span class=\"token operator\">&#061;<\/span>gc-%t.log:time<\/p>\n<p>\u4e3a\u4ec0\u4e48\u4ece 3.x \u8fc1\u8fc7\u6765\u7684\u56e2\u961f\u4f1a\u5728\u8fd9\u91cc\u8e29\u5751&#xff1a;3.x \u65f6\u4ee3\u5f88\u591a\u56e2\u961f\u7528 JDK 8\/11 &#043; ParallelGC&#xff0c;\u628a -XX:&#043;UseParallelGC \u5199\u6b7b\u5728 extraJavaOptions \u91cc\u3002\u8fc1\u5230 4.x \u540e\u8fd9\u884c\u914d\u7f6e\u4ecd\u5728\u751f\u6548\u5e76\u8986\u76d6\u9ed8\u8ba4&#xff0c;\u4e8e\u662f\u4f60\u4ee5\u4e3a\u5728\u7528 G1&#xff0c;\u5176\u5b9e\u8fd8\u5728\u7528 Parallel\u3002<\/p>\n<p><span class=\"token comment\"># \u81ea\u67e5&#xff1a;\u786e\u8ba4\u5b9e\u9645\u751f\u6548\u7684 GC<\/span><br \/>\nspark.executor.extraJavaOptions \u91cc\u6709\u6ca1\u6709 UseParallelGC \/ UseConcMarkSweepGC \/ ZGC \u5b57\u6837<br \/>\n<span class=\"token comment\"># \u6216\u8fd0\u884c\u65f6\u786e\u8ba4<\/span><br \/>\njcmd <span class=\"token operator\">&lt;<\/span>pid<span class=\"token operator\">&gt;<\/span> VM.flags <span class=\"token operator\">|<\/span> <span class=\"token function\">grep<\/span> <span class=\"token parameter variable\">-oE<\/span> <span class=\"token string\">&#039;Use[A-Za-z]*GC&#039;<\/span><\/p>\n<hr \/>\n<h3>3.3 \u5e76\u884c\u5ea6\u4e0e\u8d44\u6e90\u5206\u914d<\/h3>\n<h4>3.3.1 \u4e09\u4e2a\u4e0d\u540c\u5c42\u7ea7\u7684&#034;\u5e76\u884c\u5ea6&#034;&#xff0c;\u522b\u6df7\u4e3a\u4e00\u8c08<\/h4>\n<p>\u8fd9\u662f\u65b0\u624b\u5230\u4e2d\u7ea7\u6700\u5bb9\u6613\u6df7\u6dc6\u7684\u4e00\u7ec4\u6982\u5ff5\u3002Spark \u91cc&#034;\u5e76\u884c\u5ea6&#034;\u8fd9\u4e2a\u8bcd\u81f3\u5c11\u6307\u4e09\u4ef6\u4e8b&#xff1a;<\/p>\n<table>\n<tr>\u5c42\u7ea7\u51b3\u5b9a\u53c2\u6570\u542b\u4e49\u5f71\u54cd\u4ec0\u4e48<\/tr>\n<tbody>\n<tr>\n<td>\u8fdb\u7a0b\u7ea7<\/td>\n<td>spark.executor.instances<\/td>\n<td>\u6709\u591a\u5c11\u4e2a executor<\/td>\n<td>\u603b\u8d44\u6e90\u5360\u7528\u3001\u80fd\u540c\u65f6\u8dd1\u591a\u5c11 task<\/td>\n<\/tr>\n<tr>\n<td>\u69fd\u4f4d\u7ea7<\/td>\n<td>spark.executor.cores<\/td>\n<td>\u5355\u4e2a executor \u5185\u5e76\u53d1 task \u6570<\/td>\n<td>\u6bcf task \u5206\u5230\u7684\u5806 &#061; (M) \/ cores<\/td>\n<\/tr>\n<tr>\n<td>\u6570\u636e\u7ea7<\/td>\n<td>spark.sql.shuffle.partitions<\/td>\n<td>shuffle \u540e\u6709\u591a\u5c11\u4e2a\u5206\u533a&#xff08;&#061; task \u6570&#xff09;<\/td>\n<td>\u5355 task \u5904\u7406\u591a\u5c11\u6570\u636e<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u4e09\u8005\u7684\u5173\u7cfb&#xff1a;<\/p>\n<p>\u53ef\u7528 task \u69fd\u4f4d\u603b\u6570 &#061; executor.instances \u00d7 executor.cores<br \/>\n       \u2191 \u8fd9\u662f&#034;\u5e76\u53d1\u4e0a\u9650&#034;<\/p>\n<p>\u5f85\u8dd1 task \u603b\u6570     &#061; shuffle.partitions&#xff08;shuffle \u540e\u5404 stage&#xff09;<br \/>\n       \u2191 \u8fd9\u662f&#034;\u5de5\u4f5c\u91cf&#034;<\/p>\n<p>\u5de5\u4f5c\u91cf &gt; \u69fd\u4f4d\u6570  \u2192 \u6392\u961f&#xff0c;\u6b63\u5e38<br \/>\n\u5de5\u4f5c\u91cf &lt; \u69fd\u4f4d\u6570  \u2192 executor \u7a7a\u8f6c&#xff0c;\u6d6a\u8d39\u94b1<br \/>\n\u5355 task \u6570\u636e\u91cf   &#061; \u603b\u6570\u636e\u91cf \/ shuffle.partitions<br \/>\n       \u2191 \u8fd9\u4e2a\u503c\u51b3\u5b9a\u4f1a\u4e0d\u4f1a spill&#xff0c;\u901a\u5e38\u76ee\u6807\u533a\u95f4 128MB ~ 200MB<\/p>\n<p>\u4e3a\u4ec0\u4e48 spark.executor.cores \u4e0d\u662f\u8d8a\u5927\u8d8a\u597d&#xff1a;\u5806\u5927\u5c0f\u662f executor \u7ea7\u7684&#xff0c;\u4e0d\u968f cores \u589e\u52a0\u30028g \/ 4cores \u65f6\u6bcf\u4e2a task \u540d\u4e49\u4e0a\u80fd\u7528\u7684\u6267\u884c\u5185\u5b58\u7ea6\u4e3a 8g \u00d7 0.6 \/ 4 \u2248 1.2g&#xff1b;\u6539\u6210 8g \/ 8cores \u5c31\u53d8\u6210\u7ea6 600MB\u3002\u540c\u6837\u7684\u914d\u7f6e&#xff0c;\u53ea\u662f&#034;\u63d0\u9ad8\u5e76\u53d1&#034;&#xff0c;\u5c31\u628a\u672c\u6765\u4e0d spill \u7684\u4f5c\u4e1a\u53d8\u6210\u4e86\u5927\u91cf\u5c0f\u6587\u4ef6 spill \u7684\u4f5c\u4e1a\u3002 \u8fd9\u7c7b\u6539\u52a8\u5728 Spark UI \u4e0a\u8868\u73b0\u4e3a stage \u8017\u65f6\u4e0d\u964d\u53cd\u5347&#xff0c;\u4e14 disk spilled \u4ece 0 \u53d8\u6210\u51e0 GB\u3002<\/p>\n<h4>3.3.2 shuffle.partitions \u4e0e AQE \u7684\u914d\u5408<\/h4>\n<p>4.x \u91cc AQE \u9ed8\u8ba4\u5f00\u542f&#xff0c;coalescePartitions \u4f1a\u5728\u8fd0\u884c\u65f6\u628a\u8fc7\u5c0f\u7684\u5206\u533a\u5408\u5e76\u3002\u8fd9\u5e26\u6765\u4e00\u6761\u4e0e 3.x \u65f6\u4ee3\u76f8\u53cd\u7684\u64cd\u4f5c\u5efa\u8bae&#xff1a;<\/p>\n<p>\u53ef\u4ee5\u8bbe\u5927&#xff0c;\u4e0d\u5fc5\u8bbe\u5c0f\u3002 \u8bbe\u5927\u7684\u4ee3\u4ef7\u662f\u5408\u5e76\u524d\u591a\u4e00\u8f6e\u8c03\u5ea6\u5f00\u9500&#xff08;AQE \u5408\u5e76\u662f\u5728 driver \u4fa7\u57fa\u4e8e\u5df2\u6536\u96c6\u7684 map statistics \u505a\u7684&#xff09;&#xff0c;\u8bbe\u5c0f\u7684\u4ee3\u4ef7\u662f\u771f\u5b9e\u7684\u6570\u636e\u503e\u659c\u4e0e spill\u3002<\/p>\n<p>spark.sql.adaptive.enabled                      <span class=\"token boolean\">true<\/span>     <span class=\"token comment\"># 4.x \u9ed8\u8ba4 true<\/span><br \/>\nspark.sql.adaptive.coalescePartitions.enabled   <span class=\"token boolean\">true<\/span>     <span class=\"token comment\"># \u9ed8\u8ba4 true<\/span><br \/>\nspark.sql.adaptive.coalescePartitions.minPartitionNum   <span class=\"token number\">1<\/span>   <span class=\"token comment\"># \u9ed8\u8ba4\u5373 1<\/span><br \/>\nspark.sql.adaptive.advisoryPartitionSizeInBytes 64MB     <span class=\"token comment\"># \u5408\u5e76\u76ee\u6807\u5927\u5c0f<\/span><br \/>\nspark.sql.shuffle.partitions                    <span class=\"token number\">400<\/span>      <span class=\"token comment\"># \u523b\u610f\u8bbe\u5927&#xff0c;\u4ea4\u7ed9 AQE \u6536<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e<\/p>\n<table>\n<tr>\u53c2\u6570\u8bf4\u660e<\/tr>\n<tbody>\n<tr>\n<td>advisoryPartitionSizeInBytes<\/td>\n<td>AQE \u5408\u5e76\u5206\u533a\u7684\u76ee\u6807\u5927\u5c0f\u3002\u8fd9\u662f\u771f\u6b63\u63a7\u5236&#034;\u6700\u7ec8\u5e76\u884c\u5ea6&#034;\u7684\u65cb\u94ae&#xff0c;shuffle.partitions \u53ea\u662f\u4e0a\u9650<\/td>\n<\/tr>\n<tr>\n<td>minPartitionNum<\/td>\n<td>\u5408\u5e76\u540e\u81f3\u5c11\u4fdd\u7559\u591a\u5c11\u5206\u533a\u3002\u9ed8\u8ba4 1\u2014\u2014\u610f\u5473\u7740\u5c0f\u6570\u636e\u91cf\u65f6 AQE \u4f1a\u628a\u5e76\u884c\u5ea6\u538b\u5230\u6781\u4f4e\u3002\u751f\u4ea7\u5efa\u8bae\u6309\u961f\u5217\u914d\u989d\u8bbe\u4e00\u4e2a\u4e0b\u9650<\/td>\n<\/tr>\n<tr>\n<td>shuffle.partitions<\/td>\n<td>\u4e0a\u754c\u3002\u8bbe\u592a\u5c0f\u5219 AQE \u65e0\u6cd5\u62c6\u5206&#xff08;\u5b83\u53ea\u5408\u5e76\u4e0d\u62c6\u5206&#xff09;&#xff0c;\u503e\u659c\u5c31\u66b4\u9732\u4e0d\u51fa\u6765<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;\u4f5c\u4e1a\u8dd1\u5b8c\u540e\u5728 Spark UI \u7684 SQL tab \u91cc\u770b\u6700\u7ec8 stage \u7684 task \u6570\u3002\u82e5\u7b49\u4e8e shuffle.partitions \u8bf4\u660e AQE \u6ca1\u5408\u5e76&#xff08;\u6570\u636e\u91cf\u786e\u5b9e\u5927&#xff09;&#xff1b;\u82e5\u660e\u663e\u5c0f\u4e8e\u5b83&#xff0c;\u8bf4\u660e\u5408\u5e76\u751f\u6548\u3002 \u4e24\u8005\u90fd\u6b63\u5e38&#xff0c;\u4f46\u8981\u77e5\u9053\u662f\u54ea\u4e00\u79cd\u2014\u2014\u8fd9\u51b3\u5b9a\u4e86\u4e0b\u6b21\u6570\u636e\u91cf\u7ffb\u500d\u65f6\u4f1a\u4e0d\u4f1a\u7a81\u7136\u53d8\u6162\u3002<\/p>\n<h4>3.3.3 \u52a8\u6001\u5206\u533a\u5206\u914d\u7684\u4e09\u4e2a\u6613\u9519\u70b9<\/h4>\n<p>spark.dynamicAllocation.enabled&#061;true \u8ba9 executor \u6570\u91cf\u968f\u8d1f\u8f7d\u4f38\u7f29\u3002\u542c\u8d77\u6765\u5168\u662f\u597d\u5904&#xff0c;\u5b9e\u9645\u6709\u4e09\u4e2a\u53cd\u590d\u51fa\u73b0\u7684\u5751\u3002<\/p>\n<p>\u6613\u9519\u70b9 1&#xff1a;minExecutors \u4f1a\u8ba9\u7f29\u5bb9\u5931\u6548<\/p>\n<p>spark.dynamicAllocation.minExecutors   <span class=\"token number\">20<\/span><br \/>\nspark.dynamicAllocation.maxExecutors   <span class=\"token number\">200<\/span><\/p>\n<p>minExecutors \u662f\u4e0b\u9650&#xff0c;\u4f5c\u4e1a\u8fd0\u884c\u671f\u95f4\u8fd9 20 \u4e2a executor \u4e0d\u4f1a\u88ab\u91ca\u653e\u3002\u8bbe\u6210 20 \u610f\u5473\u7740&#xff1a;\u5373\u4f7f\u53ea\u5269\u6700\u540e\u4e00\u4e2a task \u5728\u8dd1&#xff0c;\u4f60\u4ecd\u7136\u5360\u7740 20 \u4e2a executor \u7684\u914d\u989d\u3002<\/p>\n<p>\u5b98\u65b9\u8bed\u4e49&#xff1a;minExecutors \u9ed8\u8ba4\u503c\u7b49\u4e8e initialExecutors\u3002\u4e5f\u5c31\u662f\u8bf4\u4f60\u8bbe\u4e86 initialExecutors \u800c\u6ca1\u8bbe minExecutors \u65f6&#xff0c;\u524d\u8005\u5c31\u6210\u4e86\u7f29\u5bb9\u7684\u4e0b\u9650\u3002\u8fd9\u662f&#034;\u5f00\u4e86\u52a8\u6001\u5206\u914d\u4f46\u4ece\u6765\u6ca1\u7f29\u8fc7\u5bb9&#034;\u7684\u7b2c\u4e00\u5927\u539f\u56e0\u3002<\/p>\n<p>\u6613\u9519\u70b9 2&#xff1a;\u6ca1\u6709 External Shuffle Service&#xff0c;\u52a8\u6001\u5206\u914d\u6839\u672c\u4e0d\u80fd\u5f00<\/p>\n<p>executor \u5185\u5b58\u7684 shuffle \u8f93\u51fa\u5982\u679c\u88ab\u56de\u6536\u5c31\u4e22\u4e86\u3002\u56e0\u6b64\u52a8\u6001\u5206\u914d\u5fc5\u987b\u914d\u5408\u5916\u90e8 shuffle \u670d\u52a1&#xff1a;<\/p>\n<p>spark.dynamicAllocation.enabled       <span class=\"token boolean\">true<\/span><br \/>\nspark.shuffle.service.enabled         <span class=\"token boolean\">true<\/span>       <span class=\"token comment\"># YARN&#xff1a;NM \u4e0a\u5fc5\u987b\u90e8\u7f72 ExternalShuffleService<\/span><br \/>\nspark.dynamicAllocation.shuffleTracking.enabled  <span class=\"token boolean\">false<\/span><\/p>\n<p>\u5728 K8s \u4e0a\u6ca1\u6709 shuffle service \u8fd9\u4e2a\u8fdb\u7a0b&#xff0c;\u6539\u7528 shuffle tracking&#xff08;\u7531 driver \u8ddf\u8e2a\u54ea\u4e9b executor \u8fd8\u6301\u6709\u672a\u6d88\u8d39\u7684 shuffle \u6570\u636e&#xff09;\u6216\u6301\u4e45\u5316 shuffle \u670d\u52a1&#xff1a;<\/p>\n<p><span class=\"token comment\"># K8s \u8def\u7ebf&#xff1a;driver \u8ddf\u8e2a\u672a\u6d88\u8d39 shuffle \u7684 executor&#xff0c;\u4e0d\u4f9d\u8d56\u5916\u90e8\u670d\u52a1<\/span><br \/>\nspark.dynamicAllocation.enabled                 <span class=\"token boolean\">true<\/span><br \/>\nspark.dynamicAllocation.shuffleTracking.enabled <span class=\"token boolean\">true<\/span><\/p>\n<p><span class=\"token comment\"># \u82e5\u8981\u628a shuffle \u6570\u636e\u843d\u5230\u5916\u90e8\u6301\u4e45\u5316\u5b58\u50a8&#xff08;\u8de8\u4f5c\u4e1a\u590d\u7528\u3001\u7f29\u5bb9\u4e0d\u4e22&#xff09;&#xff0c;<\/span><br \/>\n<span class=\"token comment\"># \u9700\u5f15\u5165 Remote Shuffle Service \u5e76\u8bbe\u7f6e\u5176\u63d2\u4ef6\u7c7b&#xff0c;\u4f8b\u5982 Celeborn \u5f62\u6001&#xff1a;<\/span><br \/>\n<span class=\"token comment\">#   spark.shuffle.sort.io.plugin.class &#061; &lt;\u5bf9\u5e94\u53d1\u884c\u7248\u7684 RemoteShuffleManager \u63d2\u4ef6\u7c7b&gt;<\/span><br \/>\n<span class=\"token comment\"># \u672c\u7cfb\u5217\u4e0d\u5c55\u5f00 RSS&#xff08;\u5c5e\u53ef\u9009\u7ec4\u4ef6&#xff0c;\u4e14\u63d2\u4ef6\u7c7b\u540d\u968f\u5b9e\u73b0\u4e0d\u540c&#xff09;\u3002<\/span><br \/>\n<span class=\"token comment\"># \u2605 \u6ce8\u610f&#xff1a;\u63d2\u4ef6\u7c7b\u5168\u540d\u8bf7\u4ee5\u6240\u9009\u5b9e\u73b0\u7684\u5b98\u65b9\u6587\u6863\u4e3a\u51c6&#xff0c;\u672c\u7cfb\u5217\u4e0d\u7ed9\u51fa\u672a\u7ecf\u53d6\u8bc1\u7684\u7c7b\u540d\u3002<\/span><\/p>\n<p>\u4e00\u6761\u7eaa\u5f8b&#xff1a;shuffleTracking \u4e0e shuffle.service \u4e0d\u80fd\u540c\u65f6\u5f00&#xff0c;\u4e14\u5404\u81ea\u7684\u9002\u7528\u96c6\u7fa4\u7ba1\u7406\u5668\u4e0d\u540c\u3002\u5728 YARN \u4e0a\u5f00 shuffleTracking \u800c\u5728 K8s \u4e0a\u5f00 shuffle.service&#xff0c;\u90fd\u662f\u914d\u4e86\u4e0d\u751f\u6548\u3001\u4f46\u62a5\u9519\u5f88\u665a\u624d\u51fa\u73b0\u7684\u5178\u578b\u9519\u914d\u3002<\/p>\n<p>\u6613\u9519\u70b9 3&#xff1a;\u52a8\u6001\u5206\u914d\u4e0e AQE \u5408\u5e76\u5206\u533a\u4f1a\u4e92\u76f8\u62b5\u6d88<\/p>\n<p>AQE \u5408\u5e76\u5206\u533a \u2192 \u6700\u7ec8 stage \u7684 task \u6570\u53d8\u5c11<br \/>\n       \u2193<br \/>\nexecutor \u7a7a\u95f2\u8d85\u8fc7 executorIdleTimeout&#xff08;\u9ed8\u8ba4 60s&#xff09;\u2192 \u88ab\u56de\u6536<br \/>\n       \u2193<br \/>\n\u4e0b\u4e00\u4e2a stage task \u6570\u5f88\u591a \u2192 \u91cd\u65b0\u7533\u8bf7 executor \u2192 JVM \u542f\u52a8 &#043; \u53cd\u5e8f\u5217\u5316 &#043; \u6ce8\u518c<br \/>\n       \u2193<br \/>\n&#034;\u6296\u52a8&#034;&#xff1a;executor \u6570\u5728 min\/max \u4e4b\u95f4\u53cd\u590d\u6a2a\u8df3&#xff0c;\u541e\u5410\u53cd\u800c\u4e0b\u964d<\/p>\n<p>\u5224\u636e&#xff1a;Spark UI \u7684 Executors \u9875\u770b executor \u7684 Removal Reason \u5206\u5e03\u3002\u5927\u91cf idle \u4e14\u65f6\u95f4\u70b9\u5bc6\u96c6\u4ea4\u66ff&#xff0c;\u5c31\u662f\u8fd9\u4e2a\u6a21\u5f0f\u3002<\/p>\n<p>\u5904\u7f6e&#xff08;\u6309\u4f18\u5148\u7ea7&#xff09;&#xff1a;<\/p>\n<p>spark.dynamicAllocation.executorIdleTimeout       300s   <span class=\"token comment\"># \u4ece\u9ed8\u8ba4 60s \u653e\u5927&#xff0c;\u5438\u6536 stage \u95f4\u7a7a\u7a97<\/span><br \/>\nspark.dynamicAllocation.cachedExecutorIdleTimeout 1800s  <span class=\"token comment\"># \u6301\u6709\u7f13\u5b58 block \u7684 executor \u5355\u72ec\u653e\u5bbd<\/span><br \/>\nspark.dynamicAllocation.minExecutors               <span class=\"token number\">10<\/span>    <span class=\"token comment\"># \u8bbe\u4e00\u4e2a\u771f\u5b9e\u4e0b\u9650&#xff0c;\u907f\u514d\u7f29\u5230 0 \u518d\u51b7\u542f\u52a8<\/span><\/p>\n<h4>3.3.4 \u6570\u636e\u672c\u5730\u6027&#xff1a;\u4e00\u4e2a\u7ecf\u5e38\u88ab\u9ad8\u4f30\u7684\u65cb\u94ae<\/h4>\n<p>spark.locality.wait        3s     <span class=\"token comment\"># \u9ed8\u8ba4\u503c&#xff0c;\u5bf9 PROCESS\/NODE\/RACK \u9010\u7ea7\u7b49\u5f85<\/span><br \/>\nspark.locality.wait.node   3s<br \/>\nspark.locality.wait.rack   0s     <span class=\"token comment\"># \u5efa\u8bae\u76f4\u63a5\u5173\u6389<\/span><\/p>\n<p>\u5b98\u65b9\u5728 Tuning \u9875\u7ed9\u51fa\u7684\u5efa\u8bae\u662f&#xff1a;\u5982\u679c\u96c6\u7fa4\u4e0a\u5927\u90e8\u5206 task \u7684 locality \u90fd\u662f ANY&#xff0c;\u8bf4\u660e\u6570\u636e\u5728 HDFS \u4e0a\u590d\u5236\u56e0\u5b50\u592a\u5c0f&#xff0c;\u6216\u8005 task \u592a\u591a&#xff0c;\u800c\u4e0d\u662f&#034;\u8be5\u8c03\u5927 locality.wait&#034;\u3002<\/p>\n<p>\u8f66\u8054\u7f51\u573a\u666f\u7684\u73b0\u5b9e\u5224\u65ad&#xff1a;\u6e56\u8868\u6570\u636e\u5728\u5bf9\u8c61\u5b58\u50a8&#xff08;S3\/OSS&#xff09;\u4e0a\u65f6&#xff0c;&#034;\u8282\u70b9\u672c\u5730\u6027&#034;\u8fd9\u4e2a\u6982\u5ff5\u672c\u8eab\u4e0d\u6210\u7acb\u2014\u2014\u6ca1\u6709 HDFS \u7684\u6570\u636e\u5757\u4f4d\u7f6e\u3002\u6b64\u65f6\u5e94\u5f53&#xff1a;<\/p>\n<p>spark.locality.wait        0s      <span class=\"token comment\"># \u5bf9\u8c61\u5b58\u50a8&#xff1a;\u7b49\u672c\u5730\u6027\u7eaf\u6d6a\u8d39<\/span><br \/>\nspark.sql.files.maxPartitionBytes  256MB   <span class=\"token comment\"># \u7528\u5207\u7247\u5927\u5c0f\u800c\u4e0d\u662f\u4f4d\u7f6e\u6765\u63a7\u5e76\u884c\u5ea6<\/span><\/p>\n<p>\u7b2c 2 \u7ae0\u8868 2.5.1 \u63d0\u5230 spark.sql.maxSinglePartitionBytes \u5728 4.0 \u8d77\u9ed8\u8ba4\u4ece Long.MaxValue \u6539\u4e3a 128m\u3002\u8fd9\u6761\u4e0e\u672c\u5730\u6027\u4e00\u8d77\u770b&#xff1a;4.x \u7684\u9ed8\u8ba4\u884c\u4e3a\u5df2\u7ecf\u5728\u671d&#034;\u4e0d\u4f9d\u8d56\u6570\u636e\u672c\u5730\u6027\u3001\u7528\u5207\u7247\u63a7\u5e76\u884c&#034;\u7684\u65b9\u5411\u8d70&#xff0c;\u628a locality.wait \u8c03\u5927\u662f\u9006\u7740\u8fd9\u4e2a\u65b9\u5411\u3002<\/p>\n<hr \/>\n<h3>3.4 \u4e0e Flink \u5171\u5b58&#xff1a;\u8d44\u6e90\u4e0e\u5199\u5165\u6743<\/h3>\n<h4>\u56fe 3-2 \u5171\u96c6\u7fa4\u7684\u53cc\u5c42\u9694\u79bb<\/h4>\n<p>        \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 \u7b2c\u4e00\u5c42&#xff1a;\u8d44\u6e90\u9694\u79bb&#xff08;\u9632\u62a2\u5360&#xff09;\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n        \u2502                                                          \u2502<br \/>\n        \u2502   YARN:  \u961f\u5217 real-time(900g) \u2502 batch_normal(400g)        \u2502<br \/>\n        \u2502          \u6d41\u961f\u5217 capacity \u56fa\u5b9a &#043; maximum-capacity \u4e0d\u8d85\u5356     \u2502<br \/>\n        \u2502   K8s:   namespace flink-rt   \u2502 data-batch                \u2502<br \/>\n        \u2502          ResourceQuota        \u2502 PriorityClass             \u2502<br \/>\n        \u2502          flink&#061;high           \u2502 spark&#061;batch(\u4f4e)           \u2502<br \/>\n        \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<\/p>\n<p>        \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 \u7b2c\u4e8c\u5c42&#xff1a;\u5199\u5165\u6743\u4ef2\u88c1&#xff08;\u9632\u4e22\u6570\u636e&#xff09;\u2605 \u66f4\u91cd\u8981 \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n        \u2502                                                          \u2502<br \/>\n        \u2502   \u6e56\u8868 dws_trip_segment                                   \u2502<br \/>\n        \u2502     \u251c\u2500 \u5e38\u6001&#xff1a;Flink \u6d41\u5199&#xff08;\u552f\u4e00\u63d0\u4ea4\u65b9&#xff09;                      \u2502<br \/>\n        \u2502     \u251c\u2500 \u6536\u53e3\u7a97&#xff1a;Spark \u63a5\u7ba1\u5199&#xff08;02:00-04:00&#xff09;                 \u2502<br \/>\n        \u2502     \u2514\u2500 \u4ef2\u88c1&#xff1a;\u5916\u90e8\u9501 &#043; \u65f6\u95f4\u7a97 &#043; tag \u951a\u5b9a                    \u2502<br \/>\n        \u2502                                                          \u2502<br \/>\n        \u2502   \u5931\u8d25\u6a21\u5f0f&#xff1a;\u4e24\u8fb9\u7684\u9501\u4e92\u4e0d\u53ef\u89c1 \u2192 \u540e\u63d0\u4ea4\u8005\u8986\u76d6\u524d\u8005 \u2192 \u9759\u9ed8\u4e22\u6570  \u2502<br \/>\n        \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<\/p>\n<h4>3.4.1 \u7b2c\u4e00\u5c42&#xff1a;\u8d44\u6e90\u9694\u79bb<\/h4>\n<p>YARN \u8def\u7ebf<\/p>\n<p><span class=\"token comment\"># \u6d41\u4f5c\u4e1a\u961f\u5217&#xff1a;\u5bb9\u91cf\u56fa\u5b9a\u3001\u4e0d\u5141\u8bb8\u88ab\u62a2\u5360<\/span><br \/>\nyarn.scheduler.capacity.root.realtime.capacity              <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">60<\/span><br \/>\nyarn.scheduler.capacity.root.realtime.maximum-capacity      <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">70<\/span><br \/>\nyarn.scheduler.capacity.root.realtime.user-limit-factor     <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">1<\/span><\/p>\n<p><span class=\"token comment\"># \u6279\u4f5c\u4e1a\u961f\u5217&#xff1a;\u53ef\u88ab\u538b\u5230\u4f4e\u4f18\u5148\u7ea7&#xff0c;\u5141\u8bb8\u501f\u7528\u7a7a\u95f2\u8d44\u6e90\u4f46\u4e0d\u8d8a\u754c<\/span><br \/>\nyarn.scheduler.capacity.root.batch_normal.capacity          <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">30<\/span><br \/>\nyarn.scheduler.capacity.root.batch_normal.maximum-capacity  <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">60<\/span><\/p>\n<p><span class=\"token comment\"># Spark \u4fa7\u663e\u5f0f\u843d\u5230\u4f4e\u4f18\u5148\u7ea7\u961f\u5217<\/span><br \/>\nspark.yarn.queue                    batch_normal<br \/>\nspark.yarn.priority                 <span class=\"token number\">1<\/span>        <span class=\"token comment\"># \u6570\u5b57\u5c0f &#061; \u4f18\u5148\u7ea7\u9ad8&#xff08;Capacity Scheduler \u8bed\u4e49\u9700\u6309\u53d1\u884c\u7248\u6838\u5bf9&#xff09;<\/span><\/p>\n<p>\u5173\u952e\u7ea6\u675f&#xff1a;\u6279\u961f\u5217\u7684 maximum-capacity \u5fc5\u987b &lt; 100 \u2212 \u6d41\u961f\u5217 capacity \u7684\u5b89\u5168\u4f59\u91cf\u3002\u5426\u5219\u51cc\u6668\u56de\u5237\u4efb\u52a1\u4f1a\u628a\u6d41\u4f5c\u4e1a\u7684\u5bb9\u5668\u6324\u6389&#xff0c;\u8868\u73b0\u4e3a&#034;\u767d\u5929\u4e00\u5207\u6b63\u5e38&#xff0c;\u65e9\u4e0a 3 \u70b9\u5b9e\u65f6\u94fe\u8def\u5ef6\u8fdf\u98d9\u5347&#034;\u3002<\/p>\n<p>K8s \u8def\u7ebf<\/p>\n<p><span class=\"token comment\"># \u6d41&#xff1a;\u9ad8\u4f18\u5148\u7ea7 &#043;  Guaranteed QoS<\/span><br \/>\n<span class=\"token key atrule\">apiVersion<\/span><span class=\"token punctuation\">:<\/span> scheduling.k8s.io\/v1<br \/>\n<span class=\"token key atrule\">kind<\/span><span class=\"token punctuation\">:<\/span> PriorityClass<br \/>\n<span class=\"token key atrule\">metadata<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">{<\/span> <span class=\"token key atrule\">name<\/span><span class=\"token punctuation\">:<\/span> flink<span class=\"token punctuation\">&#8211;<\/span>realtime <span class=\"token punctuation\">}<\/span><br \/>\n<span class=\"token key atrule\">value<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token number\">1000000<\/span><br \/>\n<span class=\"token key atrule\">preemptionPolicy<\/span><span class=\"token punctuation\">:<\/span> PreemptLowerPriority   <span class=\"token comment\"># \u5141\u8bb8\u62a2\u5360\u6279\u4f5c\u4e1a pod<\/span><br \/>\n<span class=\"token punctuation\">&#8212;<\/span><br \/>\n<span class=\"token comment\"># \u6279&#xff1a;\u4f4e\u4f18\u5148\u7ea7 &#043; Burstable<\/span><br \/>\n<span class=\"token key atrule\">apiVersion<\/span><span class=\"token punctuation\">:<\/span> scheduling.k8s.io\/v1<br \/>\n<span class=\"token key atrule\">kind<\/span><span class=\"token punctuation\">:<\/span> PriorityClass<br \/>\n<span class=\"token key atrule\">metadata<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">{<\/span> <span class=\"token key atrule\">name<\/span><span class=\"token punctuation\">:<\/span> spark<span class=\"token punctuation\">&#8211;<\/span>batch <span class=\"token punctuation\">}<\/span><br \/>\n<span class=\"token key atrule\">value<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token number\">1000<\/span><br \/>\n<span class=\"token key atrule\">preemptionPolicy<\/span><span class=\"token punctuation\">:<\/span> Never<\/p>\n<p>spark.kubernetes.namespace                     data-batch<br \/>\nspark.kubernetes.driver.podTemplateFile        conf\/spark-driver-batch.yaml<br \/>\nspark.kubernetes.executor.podTemplateFile      conf\/spark-executor-batch.yaml<br \/>\n<span class=\"token comment\"># \u4e24\u4e2a template \u91cc\u8bbe&#xff1a;priorityClassName: spark-batch<\/span><br \/>\n<span class=\"token comment\">#                    requests.memory &#061;&#061; limits.memory&#xff08;\u907f\u514d\u88ab\u9a71\u9010&#xff09;<\/span><\/p>\n<p>K8s \u7279\u6709\u7684\u9a71\u9010\u98ce\u9669&#xff1a;executor pod \u82e5\u53ea\u8bbe requests \u4e0d\u8bbe limits&#xff0c;\u8282\u70b9\u5185\u5b58\u538b\u529b\u65f6\u5b83\u6bd4 Flink \u7684 Guaranteed pod \u5148\u88ab\u9a71\u9010\u3002\u8868\u73b0\u4e3a&#034;Spark \u4f5c\u4e1a\u968f\u673a\u4e22 executor\u3001\u91cd\u8dd1 task&#034;&#xff0c;\u800c YARN \u4e0a\u4e0d\u4f1a\u6709\u8fd9\u4e2a\u73b0\u8c61\u3002<\/p>\n<h4>3.4.2 \u7b2c\u4e8c\u5c42&#xff1a;\u5199\u5165\u6743\u4ef2\u88c1&#xff08;\u6bd4\u8d44\u6e90\u91cd\u8981&#xff09;<\/h4>\n<p>\u8d44\u6e90\u4e89\u62a2\u7684\u540e\u679c\u662f\u6162&#xff0c;\u5199\u5165\u6743\u51b2\u7a81\u7684\u540e\u679c\u662f\u6570\u636e\u4e22\u5931\u3002\u540e\u8005\u4e25\u91cd\u5f97\u591a&#xff0c;\u800c\u4e14\u4e0d\u7559\u9519\u8bef\u3002<\/p>\n<p>\u4e3a\u4ec0\u4e48 Spark \u4fa7\u7279\u522b\u5371\u9669&#xff1a;\u6279\u4f5c\u4e1a\u5931\u8d25\u91cd\u8dd1\u662f\u5e38\u6001\u3002\u4e00\u4e2a\u6b63\u5728\u8dd1\u7684 Flink \u6d41\u5199\u4f5c\u4e1a&#xff0c;\u88ab\u4e00\u4e2a&#034;\u91cd\u8bd5\u4e86 3 \u6b21\u7684 Spark \u56de\u5237\u4f5c\u4e1a&#034;\u63d0\u4ea4\u4e86\u65b0\u5feb\u7167&#xff0c;Flink \u4fa7\u7684\u4e0b\u4e00\u4e2a checkpoint \u4f1a\u57fa\u4e8e\u9519\u8bef\u7684\u57fa\u7ebf\u7ee7\u7eed\u2014\u2014\u6ca1\u6709\u4efb\u4f55\u4e00\u65b9\u4f1a\u62a5\u9519\u3002<\/p>\n<p>\u4e09\u5c42\u4ef2\u88c1&#xff0c;\u6309\u987a\u5e8f\u843d\u5730&#xff1a;<\/p>\n<p>\u2460 \u5916\u90e8\u9501&#xff08;\u5fc5\u987b&#xff09;<br \/>\n   \u5f15\u64ce\u81ea\u7ba1\u7684\u9501\u53ea\u8986\u76d6\u81ea\u5df1\u3002Flink \u7684\u9501\u770b\u4e0d\u89c1 Spark&#xff0c;\u53cd\u4e4b\u4ea6\u7136\u3002<br \/>\n   \u2192 Iceberg \u914d JDBC \/ ZK \u9501&#xff1b;Paimon \u4fa7\u9760\u5355\u4e00\u63d0\u4ea4\u65b9\u7ea6\u5b9a<\/p>\n<p>\u2461 \u65f6\u95f4\u7a97&#xff08;\u7ea6\u5b9a&#xff09;<br \/>\n   Spark \u53ea\u5728 02:00-04:00 \u6301\u6709\u5199\u5165\u6743&#xff1b;\u7a97\u53e3\u5916\u4e00\u5f8b\u53ea\u8bfb<br \/>\n   \u2192 \u7a97\u53e3\u7531\u8c03\u5ea6\u5668\u4fdd\u8bc1&#xff0c;\u4e0d\u7531\u4f5c\u4e1a\u81ea\u5df1\u5224\u65ad&#xff08;\u4f5c\u4e1a\u53ef\u80fd\u8d85\u65f6\u8de8\u7a97&#xff09;<\/p>\n<p>\u2462 tag \u951a\u5b9a &#043; \u539f\u5b50\u5207\u6362&#xff08;\u6280\u672f\u515c\u5e95&#xff09;<br \/>\n   \u63a5\u7ba1\u524d&#xff1a;\u7ed9\u8868\u6253 tag takeover-&lt;jobid&gt;&#xff0c;\u4f5c\u4e3a\u56de\u6eda\u70b9<br \/>\n   \u63a5\u7ba1\u4e2d&#xff1a;Flink \u4fa7\u7ed3\u679c\u5199\u5f71\u5b50\u8868&#xff0c;\u4e0d\u76f4\u63a5\u5199\u76ee\u6807\u8868<br \/>\n   \u7ed3\u675f\u65f6&#xff1a;Spark \u5b8c\u6210 \u2192 \u5bf9\u8d26\u901a\u8fc7 \u2192 \u539f\u5b50\u5207\u6362 \u2192 Flink \u6062\u590d\u76f4\u5199<\/p>\n<p>Iceberg \u5916\u90e8\u9501\u914d\u7f6e\u793a\u4f8b<\/p>\n<p>spark.sql.catalog.iceberg.catalog-impl              org.apache.iceberg.jdbc.JdbcCatalog<br \/>\nspark.sql.catalog.iceberg.property.dialect          default<br \/>\n# \u5173\u952e&#xff1a;\u9501\u8868\u4e0e\u5143\u6570\u636e\u5e93\u540c\u5e93&#xff0c;\u4e24\u4e2a\u5f15\u64ce\u5fc5\u987b\u6307\u5411\u540c\u4e00 URI<br \/>\nspark.sql.catalog.iceberg.uri                       jdbc:mysql:\/\/mysql-host:3306\/iceberg_meta<br \/>\nspark.sql.catalog.iceberg.lock-type                 zookeeper<br \/>\nspark.sql.catalog.iceberg.lock-zookeeper-address    zk1:2181,zk2:2181,zk3:2181<\/p>\n<p>Flink \u4fa7\u5fc5\u987b\u5b8c\u5168\u4e00\u81f4&#xff1a;<\/p>\n<p><span class=\"token comment\"># Flink SQL<\/span><br \/>\nCREATE CATALOG iceberg WITH (<br \/>\n  &#039;type&#039; &#061; &#039;iceberg&#039;<span class=\"token punctuation\">,<\/span><br \/>\n  &#039;catalog<span class=\"token punctuation\">&#8211;<\/span>type&#039; &#061; &#039;jdbc&#039;<span class=\"token punctuation\">,<\/span><br \/>\n  &#039;uri&#039; &#061; &#039;jdbc<span class=\"token punctuation\">:<\/span>mysql<span class=\"token punctuation\">:<\/span>\/\/mysql<span class=\"token punctuation\">&#8211;<\/span>host<span class=\"token punctuation\">:<\/span>3306\/iceberg_meta&#039;<span class=\"token punctuation\">,<\/span>   <span class=\"token punctuation\">&#8211;<\/span><span class=\"token punctuation\">&#8211;<\/span> \u4e0e Spark \u540c\u4e00\u4e2a<br \/>\n  &#039;lock.type&#039; &#061; &#039;zookeeper&#039;<span class=\"token punctuation\">,<\/span><br \/>\n  &#039;lock.zookeeper.address&#039; &#061; &#039;zk1<span class=\"token punctuation\">:<\/span><span class=\"token number\">2181<\/span><span class=\"token punctuation\">,<\/span>zk2<span class=\"token punctuation\">:<\/span><span class=\"token number\">2181<\/span><span class=\"token punctuation\">,<\/span>zk3<span class=\"token punctuation\">:<\/span>2181&#039; <span class=\"token punctuation\">&#8211;<\/span><span class=\"token punctuation\">&#8211;<\/span> \u4e0e Spark \u540c\u4e00\u4e2a<br \/>\n);<\/p>\n<p>\u9a8c\u6536\u6807\u51c6&#xff08;\u8fd9\u6761\u5fc5\u987b\u5b9e\u6d4b&#xff0c;\u4e0d\u80fd\u9760 review&#xff09;<\/p>\n<p>\u6d4b\u8bd5\u6b65\u9aa4&#xff1a;<br \/>\n  1. \u8d77\u4e00\u4e2a Flink \u6d41\u5199\u4f5c\u4e1a&#xff0c;\u6301\u7eed\u5199 dws_trip_segment<br \/>\n  2. \u540c\u65f6\u8d77\u4e00\u4e2a Spark \u6279\u4f5c\u4e1a&#xff0c;\u5bf9\u540c\u4e00\u5f20\u8868\u6267\u884c INSERT<br \/>\n  3. \u89c2\u5bdf\u662f\u5426\u6709\u4e00\u65b9\u6536\u5230 commit \u51b2\u7a81\u5f02\u5e38<\/p>\n<p>\u9884\u671f&#xff08;\u9501\u751f\u6548&#xff09;&#xff1a;\u540e\u5230\u8005\u629b CommitFailedException \u5e76\u91cd\u8bd5&#xff0c;\u6216\u7b49\u5f85\u9501<br \/>\n\u5b9e\u9645&#xff08;\u9501\u672a\u751f\u6548&#xff09;&#xff1a;\u4e24\u8fb9\u90fd&#034;\u6210\u529f&#034;&#xff0c;\u4f46\u5176\u4e2d\u4e00\u65b9\u7684\u6570\u636e\u6d88\u5931<br \/>\n                 \u2192 \u7528 SELECT count(*) FROM t.snapshots \u770b\u4e24\u4e2a\u5feb\u7167\u7684 parent \u662f\u5426\u5206\u53c9<\/p>\n<p>\u5982\u679c\u8fd9\u4e2a\u6d4b\u8bd5\u6ca1\u901a\u8fc7&#xff0c;\u4e0d\u8981\u4e0a\u751f\u4ea7\u3002 \u524d\u4f5c\u5728 Flink &#043; Kafka Connect \u573a\u666f\u8bb0\u5f55\u8fc7\u540c\u7c7b\u95ee\u9898&#xff0c;\u7ed3\u8bba\u662f&#034;\u540c\u4e00\u5f20\u8868\u4e0d\u53ef\u540c\u65f6\u7531\u4e24\u4e2a\u72ec\u7acb\u63d0\u4ea4\u65b9\u5199\u5165&#xff0c;\u5fc5\u987b\u5f71\u5b50\u8868\u9694\u79bb&#034;\u3002Spark \u52a0\u5165\u540e&#xff0c;\u8fd9\u4e2a\u7ea6\u675f\u7684\u89e6\u53d1\u6982\u7387\u663e\u8457\u4e0a\u5347\u2014\u2014\u56e0\u4e3a\u6279\u4f5c\u4e1a\u7684\u91cd\u8bd5\u662f\u8bbe\u8ba1\u7684\u4e00\u90e8\u5206\u3002<\/p>\n<h4>3.4.3 \u4e0e Flink \u7684\u8bed\u4e49\u5bf9\u7167&#xff08;\u907f\u514d\u62ff\u9519\u5730\u56fe&#xff09;<\/h4>\n<table>\n<tr>\u7ef4\u5ea6FlinkSpark\u8fc1\u79fb\u65f6\u6700\u5bb9\u6613\u51fa\u9519\u7684\u5730\u65b9<\/tr>\n<tbody>\n<tr>\n<td>\u8d44\u6e90\u5355\u4f4d<\/td>\n<td>Slot \/ TaskManager<\/td>\n<td>Executor \/ Task<\/td>\n<td>\u65e0\u6cd5\u6309&#034;\u6838\u6570&#034;\u76f4\u63a5\u6362\u7b97&#xff1b;Slot \u5171\u4eab\u7ec4\u5728 Spark \u65e0\u5bf9\u5e94\u6982\u5ff5<\/td>\n<\/tr>\n<tr>\n<td>\u5185\u5b58\u6a21\u578b<\/td>\n<td>TM \u5185\u5b58\u5206\u6bb5&#xff08;heap\/managed\/network\/state&#xff09;<\/td>\n<td>\u5806 &#043; \u7edf\u4e00\u533a M &#043; overhead<\/td>\n<td>Flink \u7684 managed memory \u4e0e Spark \u7684 off-heap \u4e0d\u662f\u4e00\u56de\u4e8b<\/td>\n<\/tr>\n<tr>\n<td>\u5927\u72b6\u6001<\/td>\n<td>RocksDB \/ ForSt&#xff0c;48h TTL<\/td>\n<td>\u6d41\u72b6\u6001\u968f checkpoint&#xff0c;\u6279\u65e0\u72b6\u6001<\/td>\n<td>\u6709\u72b6\u6001\u7a97\u53e3\u4f5c\u4e1a\u4e0d\u8981\u642c&#xff08;\u7b2c 1 \u7ae0\u7ea2\u7ebf&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u9694\u79bb\u624b\u6bb5<\/td>\n<td>slotSharingGroup<\/td>\n<td>\u961f\u5217 \/ namespace<\/td>\n<td>Spark \u4fa7\u6ca1\u6709&#034;\u540c\u4f5c\u4e1a\u5185\u4e0d\u540c\u7b97\u5b50\u9694\u79bb&#034;\u7684\u80fd\u529b<\/td>\n<\/tr>\n<tr>\n<td>\u4e00\u81f4\u6027<\/td>\n<td>Checkpoint &#043; 2PC<\/td>\n<td>Checkpoint &#043; \u5e42\u7b49\/2PC sink<\/td>\n<td>\u90fd\u8981\u6c42\u5168\u94fe\u8def\u7ec4\u4ef6\u652f\u6301&#xff0c;\u4efb\u4e00\u73af\u65ad\u5219\u964d\u7ea7<\/td>\n<\/tr>\n<tr>\n<td>\u5931\u8d25\u6062\u590d<\/td>\n<td>Savepoint&#xff08;\u4e0d\u517c\u5bb9\u53d8\u66f4\u9700\u65e0\u72b6\u6001\u91cd\u542f&#xff09;<\/td>\n<td>\u4ece checkpoint \u6062\u590d&#xff0c;\u4e0d\u517c\u5bb9\u9700\u6e05\u4f4d\u70b9<\/td>\n<td>\u98ce\u9669\u540c\u6e90&#xff1a;\u6267\u884c\u8ba1\u5212\u53d8\u4e86\u72b6\u6001\u5c31\u5bf9\u4e0d\u4e0a<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>3.5 History Server\u3001\u4e8b\u4ef6\u65e5\u5fd7\u4e0e\u4f5c\u4e1a\u5f52\u6863<\/h3>\n<h4>3.5.1 \u4e3a\u4ec0\u4e48\u8fd9\u662f\u7b2c 3 \u7ae0\u800c\u4e0d\u662f\u7b2c 17 \u7ae0<\/h4>\n<p>\u7b2c 17 \u7ae0\u8bb2\u53ef\u89c2\u6d4b\u4e0e\u8c03\u4f18&#xff0c;\u4f46\u4e8b\u4ef6\u65e5\u5fd7\u5fc5\u987b\u5728\u642d\u73af\u5883\u7684\u5f53\u5929\u5c31\u6253\u5f00\u3002\u539f\u56e0\u5f88\u786c&#xff1a;<\/p>\n<p>\u4e8b\u4ef6\u65e5\u5fd7\u4e0d\u53ef\u8ffd\u6eaf\u751f\u6210\u3002<br \/>\n\u6ca1\u5f00 \u2192 \u90a3\u6b21\u4f5c\u4e1a\u7684\u6240\u6709 Stage\/Task\/Shuffle\/Spill \u7ec6\u8282\u6c38\u4e45\u4e22\u5931\u3002<br \/>\n\u7b2c 17 \u7ae0\u7684 Spark UI \u590d\u76d8\u80fd\u529b&#xff0c;100% \u5efa\u7acb\u5728\u8fd9\u4efd\u65e5\u5fd7\u4e0a\u3002<\/p>\n<h4>3.5.2 \u914d\u7f6e<\/h4>\n<p><span class=\"token comment\"># \u5199\u4fa7&#xff08;\u6bcf\u4e2a\u4f5c\u4e1a&#xff0c;\u653e spark-defaults.conf&#xff09;<\/span><br \/>\nspark.eventLog.enabled          <span class=\"token boolean\">true<\/span><br \/>\nspark.eventLog.dir              hdfs:\/\/\/spark-history      <span class=\"token comment\"># \u6216 s3a:\/\/bucket\/spark-history<\/span><br \/>\nspark.eventLog.compress         <span class=\"token boolean\">true<\/span>                       <span class=\"token comment\"># \u751f\u4ea7\u5efa\u8bae\u5f00&#xff08;lz4 \u9ed8\u8ba4&#xff09;<\/span><br \/>\nspark.eventLog.logStageExecutorMetrics  <span class=\"token boolean\">true<\/span>               <span class=\"token comment\"># Stage \u7ea7 executor \u6307\u6807<\/span><br \/>\nspark.eventLog.logStageTaskMetrics      <span class=\"token boolean\">true<\/span><\/p>\n<p><span class=\"token comment\"># \u8bfb\u4fa7&#xff08;History Server \u72ec\u7acb\u8fdb\u7a0b&#xff09;<\/span><br \/>\nspark.history.fs.logDirectory   hdfs:\/\/\/spark-history<br \/>\nspark.history.ui.port           <span class=\"token number\">18080<\/span><br \/>\nspark.history.fs.cleaner.enabled   <span class=\"token boolean\">true<\/span><br \/>\nspark.history.fs.cleaner.maxAge    30d<br \/>\nspark.history.fs.numReplayThreads  <span class=\"token number\">8<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e<\/p>\n<table>\n<tr>\u53c2\u6570\u4e3a\u4ec0\u4e48<\/tr>\n<tbody>\n<tr>\n<td>eventLog.compress<\/td>\n<td>20 \u4ebf\u884c\u7684\u4f5c\u4e1a\u4e8b\u4ef6\u65e5\u5fd7\u53ef\u4ee5\u5230 GB \u7ea7\u3002\u4e0d\u5f00\u538b\u7f29&#xff0c;HDFS NameNode \u4e0e\u5c0f\u6587\u4ef6\u95ee\u9898\u4f1a\u5148\u4e8e\u67e5\u8be2\u95ee\u9898\u51fa\u73b0<\/td>\n<\/tr>\n<tr>\n<td>logStageExecutorMetrics<\/td>\n<td>\u4e0d\u5f00\u5219 UI \u91cc\u770b\u4e0d\u5230 per-executor \u7684 GC time \/ spill \u66f2\u7ebf\u2014\u2014\u7b2c 17 \u7ae0\u5b9a\u4f4d\u5bb9\u5668\u88ab\u6740\u5168\u9760\u8fd9\u4e24\u6761\u66f2\u7ebf<\/td>\n<\/tr>\n<tr>\n<td>history.fs.cleaner.enabled<\/td>\n<td>\u9ed8\u8ba4\u5173\u95ed\u3002\u4e0d\u6e05\u7406&#xff0c;\u534a\u5e74\u540e History Server \u542f\u52a8\u8981 replay \u5168\u90e8\u5386\u53f2&#xff0c;\u91cd\u542f\u4e00\u6b21\u8981\u51e0\u5341\u5206\u949f<\/td>\n<\/tr>\n<tr>\n<td>numReplayThreads<\/td>\n<td>\u4e0e cleaner \u914d\u5408\u3002\u5386\u53f2\u5e94\u7528\u591a\u65f6&#xff0c;\u5355\u7ebf\u7a0b replay \u4f1a\u8ba9 UI \u957f\u65f6\u95f4&#034;loading&#034;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>3.5.3 \u76ee\u5f55\u89c4\u5212&#xff1a;\u4e00\u4e2a\u88ab\u5ffd\u89c6\u7684\u6210\u672c<\/h4>\n<p>hdfs:\/\/\/spark-history\/<br \/>\n\u251c\u2500\u2500 done\/          \u2190 \u5df2\u7ed3\u675f\u4f5c\u4e1a&#xff0c;History Server \u4ece\u8fd9\u91cc\u8bfb<br \/>\n\u2514\u2500\u2500 in-progress\/   \u2190 \u8fd0\u884c\u4e2d\u4f5c\u4e1a&#xff08;\u9700\u5f00 spark.history.retainedApplications&#xff09;<\/p>\n<p>spark.history.fs.logDirectory        hdfs:\/\/\/spark-history\/done<br \/>\nspark.eventLog.dir                   hdfs:\/\/\/spark-history\/in-progress<br \/>\nspark.history.fs.update.interval     10s<\/p>\n<p>\u4e3a\u4ec0\u4e48\u5206\u4e24\u4e2a\u76ee\u5f55&#xff1a;History Server \u53ea\u6709\u4f5c\u4e1a\u7ed3\u675f\u540e\u624d\u89e3\u6790 done\/\u3002\u82e5\u4e24\u76ee\u5f55\u5408\u4e00&#xff0c;\u8fd0\u884c\u4e2d\u7684\u4f5c\u4e1a\u5728 UI \u4e0a\u4e0d\u53ef\u89c1&#xff08;\u6216\u663e\u793a\u4e0d\u5b8c\u6574&#xff09;\u3002\u5206\u5f00\u540e&#xff0c;\u8fd0\u884c\u4e2d\u4f5c\u4e1a\u770b live UI&#xff08;driver \u7684 4040&#xff09;&#xff0c;\u7ed3\u675f\u540e\u770b History Server&#xff0c;\u4e24\u8005\u6570\u636e\u4e00\u81f4\u3002<\/p>\n<p>driver \u7684 4040 \u7aef\u53e3\u5728 cluster \u6a21\u5f0f\u4e0b\u5728\u5bb9\u5668\u91cc&#xff0c;\u9700\u8981 YARN proxy \u6216 kubectl port-forward \u624d\u80fd\u8bbf\u95ee\u3002\u8fd9\u662f&#034;cluster \u6a21\u5f0f\u770b\u4e0d\u5230\u5b9e\u65f6 UI&#034;\u7684\u771f\u5b9e\u539f\u56e0&#xff0c;\u4e0d\u662f\u914d\u7f6e\u9519\u4e86\u3002<\/p>\n<h4>3.5.4 \u4e8b\u4ef6\u65e5\u5fd7\u7684\u7248\u672c\u517c\u5bb9\u6027&#xff08;\u4e00\u4e2a\u4f1a\u54ac\u4eba\u7684\u7ec6\u8282&#xff09;<\/h4>\n<p>\u4e8b\u4ef6\u65e5\u5fd7\u683c\u5f0f\u968f Spark \u5927\u7248\u672c\u6f14\u8fdb\u3002Spark 4 \u7684 History Server \u80fd\u8bfb 3.5 \u7684\u65e5\u5fd7&#xff0c;\u4f46\u53cd\u8fc7\u6765\u4e0d\u884c&#xff08;\u65e7 server \u8bfb\u5230\u65b0 schema \u4f1a replay \u5931\u8d25&#xff09;\u3002<\/p>\n<p>\u8fc1\u79fb\u671f\u7684\u6b63\u786e\u987a\u5e8f&#xff1a;<br \/>\n  1. \u5148\u5347\u7ea7 History Server \u5230 4.x&#xff08;\u5411\u540e\u517c\u5bb9\u8bfb\u65e7\u65e5\u5fd7&#xff09;<br \/>\n  2. \u518d\u9010\u4e2a\u5347\u7ea7\u4f5c\u4e1a&#xff08;\u4ea7\u751f\u65b0\u683c\u5f0f\u65e5\u5fd7&#xff09;<br \/>\n  3. \u4fdd\u7559 3.5 \u65f6\u671f\u7684 done\/ \u76ee\u5f55&#xff0c;\u4e0d\u5220\u9664<br \/>\n\u9519\u8bef\u987a\u5e8f&#xff1a;\u5148\u5347\u4f5c\u4e1a\u540e\u5347 server \u2192 \u5386\u53f2 UI \u6574\u7247\u6253\u4e0d\u5f00&#xff0c;\u4e14\u62a5\u9519\u662f replay \u5f02\u5e38&#xff0c;<br \/>\n          \u770b\u8d77\u6765\u50cf\u65e5\u5fd7\u635f\u574f&#xff0c;\u5bb9\u6613\u8bef\u5220\u76ee\u5f55&#xff08;\u4e0d\u53ef\u6062\u590d&#xff09;<\/p>\n<p>\u9a8c\u6536\u6807\u51c6<\/p>\n<p><span class=\"token comment\"># 1. \u5199\u4fa7\u786e\u8ba4\u65e5\u5fd7\u751f\u6210<\/span><br \/>\nhdfs dfs <span class=\"token parameter variable\">-ls<\/span> \/spark-history\/in-progress <span class=\"token operator\">|<\/span> <span class=\"token function\">head<\/span><br \/>\n<span class=\"token comment\"># \u9884\u671f&#xff1a;\u6bcf\u4e2a\u8fd0\u884c\u4e2d\u4f5c\u4e1a\u4e00\u4e2a\u6587\u4ef6&#xff0c;\u5927\u5c0f\u968f\u65f6\u95f4\u589e\u957f<\/span><\/p>\n<p><span class=\"token comment\"># 2. \u7ed3\u675f\u540e\u786e\u8ba4\u5f52\u6863<\/span><br \/>\nhdfs dfs <span class=\"token parameter variable\">-ls<\/span> \/spark-history\/done <span class=\"token operator\">|<\/span> <span class=\"token function\">head<\/span><br \/>\n<span class=\"token comment\"># \u9884\u671f&#xff1a;\u4f5c\u4e1a FINISHED \u540e\u6587\u4ef6\u51fa\u73b0\u5728\u6b64\u76ee\u5f55<\/span><\/p>\n<p><span class=\"token comment\"># 3. History Server \u53ef\u89c1<\/span><br \/>\n<span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-s<\/span> http:\/\/history-host:18080\/json\/version <span class=\"token operator\">|<\/span> <span class=\"token function\">head<\/span><br \/>\n<span class=\"token comment\"># \u9884\u671f&#xff1a;\u8fd4\u56de JSON&#xff0c;\u4e14 UI \u5217\u8868\u80fd\u770b\u5230\u521a\u5b8c\u6210\u7684\u4f5c\u4e1a<\/span><\/p>\n<hr \/>\n<h3>3.6 \u751f\u4ea7\u4e0a\u7ebf checklist&#xff08;15 \u9879&#xff09;<\/h3>\n<p>\u8fd9\u4e00\u8282\u53ef\u4ee5\u76f4\u63a5\u6253\u5370\u5e26\u53bb\u8bc4\u5ba1\u4f1a\u3002\u6bcf\u9879\u90fd\u662f&#034;\u4e0d\u6ee1\u8db3\u5c31\u4e0d\u8be5\u4e0a\u7ebf&#034;\u7684\u7ea7\u522b\u3002<\/p>\n<table>\n<tr>#\u68c0\u67e5\u9879\u600e\u4e48\u9a8c\u4e0d\u6ee1\u8db3\u7684\u540e\u679c<\/tr>\n<tbody>\n<tr>\n<td>1<\/td>\n<td>\u90e8\u7f72\u6a21\u5f0f\u4e3a cluster<\/td>\n<td>spark-submit \u53c2\u6570\u6216 defaults<\/td>\n<td>\u63d0\u4ea4\u673a\u4e0b\u7ebf\u5bfc\u81f4\u4f5c\u4e1a\u5931\u8054<\/td>\n<\/tr>\n<tr>\n<td>2<\/td>\n<td>\u4e09\u4ef6\u5957\u7248\u672c\u4e0e artifact \u540e\u7f00\u4e00\u81f4<\/td>\n<td>spark-submit &#8211;version &#043; ls $SPARK_HOME\/jars<\/td>\n<td>\u8fd0\u884c\u671f NoSuchMethodError<\/td>\n<\/tr>\n<tr>\n<td>3<\/td>\n<td>classpath \u65e0 Scala 2.12 \u6df7\u88c5<\/td>\n<td>\u7b2c 2 \u7ae0\u56fe 2-2 \u81ea\u68c0\u547d\u4ee4<\/td>\n<td>\u8be1\u5f02\u7c7b\u578b\u8f6c\u6362\u5f02\u5e38&#xff0c;\u6392\u67e5\u6210\u672c\u6781\u9ad8<\/td>\n<\/tr>\n<tr>\n<td>4<\/td>\n<td>\u5bb9\u5668\u603b\u989d\u5df2\u6309&#034;\u5806 &#043; off-heap &#043; overhead&#034;\u6838\u7b97<\/td>\n<td>\u624b\u7b97\u5e76\u4e0e\u961f\u5217\u914d\u989d\u6bd4\u5bf9<\/td>\n<td>\u5bb9\u5668\u88ab\u7ba1\u7406\u5668\u6740&#xff0c;\u4e14\u65e5\u5fd7\u4e0d\u6307\u5411\u771f\u56e0<\/td>\n<\/tr>\n<tr>\n<td>5<\/td>\n<td>\u4e8b\u4ef6\u65e5\u5fd7\u5df2\u5f00\u542f\u4e14\u76ee\u5f55\u53ef\u5199<\/td>\n<td>\u8dd1\u4e00\u4e2a\u4f5c\u4e1a&#xff0c;hdfs dfs -ls \u786e\u8ba4<\/td>\n<td>\u4e8b\u6545\u540e\u65e0\u6cd5\u590d\u76d8<\/td>\n<\/tr>\n<tr>\n<td>6<\/td>\n<td>History Server \u7248\u672c \u2265 \u4f5c\u4e1a\u7248\u672c<\/td>\n<td>curl :18080\/json\/version<\/td>\n<td>\u5386\u53f2 UI \u6253\u4e0d\u5f00&#xff0c;\u8bef\u5224\u4e3a\u65e5\u5fd7\u635f\u574f<\/td>\n<\/tr>\n<tr>\n<td>7<\/td>\n<td>\u961f\u5217 \/ namespace \u4e0e Flink \u9694\u79bb\u4e14\u914d\u989d\u7559\u4f59\u91cf<\/td>\n<td>\u67e5\u8c03\u5ea6\u5668\u914d\u7f6e\u4e0e maximum-capacity<\/td>\n<td>\u6279\u4f5c\u4e1a\u62a2\u5360\u6d41\u4f5c\u4e1a&#xff0c;\u5b9e\u65f6\u94fe\u8def\u51cc\u6668\u5ef6\u8fdf<\/td>\n<\/tr>\n<tr>\n<td>8<\/td>\n<td>\u6e56\u8868\u5199\u5165\u6743\u4ef2\u88c1\u5df2\u5b9e\u6d4b\u901a\u8fc7<\/td>\n<td>3.4.2 \u7684\u53cc\u5199\u51b2\u7a81\u6d4b\u8bd5<\/td>\n<td>\u9759\u9ed8\u6570\u636e\u4e22\u5931<\/td>\n<\/tr>\n<tr>\n<td>9<\/td>\n<td>\u5916\u90e8\u9501\u914d\u7f6e\u4e24\u5f15\u64ce\u5b8c\u5168\u4e00\u81f4<\/td>\n<td>\u6bd4\u5bf9 URI \u4e0e lock \u53c2\u6570\u9010\u5b57\u7b26<\/td>\n<td>\u540c\u4e0a<\/td>\n<\/tr>\n<tr>\n<td>10<\/td>\n<td>\u52a8\u6001\u5206\u914d\u7684 minExecutors \u662f\u771f\u5b9e\u4e0b\u9650<\/td>\n<td>\u786e\u8ba4\u7f29\u5bb9\u5b9e\u9645\u53d1\u751f&#xff08;Executors \u9875 Removal Reason&#xff09;<\/td>\n<td>\u914d\u989d\u88ab\u957f\u671f\u5360\u4f4f&#xff0c;\u6210\u672c\u7ffb\u500d<\/td>\n<\/tr>\n<tr>\n<td>11<\/td>\n<td>shuffle.partitions \u4e0e AQE advisory size \u5df2\u6309\u6570\u636e\u91cf\u8bbe\u5b9a<\/td>\n<td>\u770b\u6700\u7ec8 stage task \u6570<\/td>\n<td>\u5c0f\u6587\u4ef6\u7206\u70b8\u6216\u957f\u5c3e task<\/td>\n<\/tr>\n<tr>\n<td>12<\/td>\n<td>GC \u7b56\u7565\u786e\u8ba4\u5b9e\u9645\u751f\u6548&#xff08;\u672a\u88ab 3.x \u9057\u7559\u53c2\u6570\u8986\u76d6&#xff09;<\/td>\n<td>jcmd &lt;pid&gt; VM.flags<\/td>\n<td>\u4ee5\u4e3a\u7528 G1 \u5b9e\u9645\u7528 Parallel<\/td>\n<\/tr>\n<tr>\n<td>13<\/td>\n<td>\u4f5c\u4e1a\u5e26 owner \/ team \/ sla tag<\/td>\n<td>\u8c03\u5ea6\u5668\u4e0e\u8d44\u6e90\u7ba1\u7406\u5668 UI \u53ef\u89c1<\/td>\n<td>\u4e8b\u6545\u65f6\u627e\u4e0d\u5230\u8d23\u4efb\u4eba&#xff0c;\u6210\u672c\u65e0\u6cd5\u5f52\u56e0<\/td>\n<\/tr>\n<tr>\n<td>14<\/td>\n<td>\u5931\u8d25\u91cd\u8bd5\u7b56\u7565\u4e0e\u5e42\u7b49\u5199\u5165\u5df2\u786e\u8ba4<\/td>\n<td>\u624b\u52a8 kill \u4e00\u6b21\u4f5c\u4e1a\u5e76\u8ba9\u5b83\u81ea\u52a8\u91cd\u8dd1<\/td>\n<td>\u91cd\u8bd5\u5bfc\u81f4\u91cd\u590d\u6570\u636e\u6216\u5199\u5165\u6743\u51b2\u7a81<\/td>\n<\/tr>\n<tr>\n<td>15<\/td>\n<td>\u56de\u6eda\u9884\u6848&#xff1a;\u6e56\u8868 tag \u4e0e\u5f71\u5b50\u8868\u5207\u6362\u8def\u5f84\u5df2\u6f14\u7ec3<\/td>\n<td>\u5b9e\u9645\u6267\u884c\u4e00\u6b21\u56de\u6eda<\/td>\n<td>\u51fa\u9519\u65f6\u53ea\u80fd\u505c\u94fe\u8def\u4eba\u5de5\u4fee<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u7b2c 8 \u9879\u662f\u8fd9\u5f20\u8868\u91cc\u552f\u4e00&#034;\u540e\u679c\u662f\u4e22\u6570\u636e&#034;\u800c\u975e&#034;\u540e\u679c\u662f\u53d8\u6162\/\u5931\u8d25&#034;\u7684\u9879\u3002\u5982\u679c\u65f6\u95f4\u53ea\u591f\u9a8c\u4e00\u9879&#xff0c;\u9a8c\u5b83\u3002<\/p>\n<hr \/>\n<h3>\u5de5\u7a0b\u8981\u70b9<\/h3>\n<li>\n<p>spark.executor.memory \u4e0d\u662f\u5bb9\u5668\u5927\u5c0f\u3002 \u5bb9\u5668 &#061; \u5806 &#043; off-heap &#043; overhead\u3002\u6240\u6709&#034;\u5bb9\u5668\u88ab\u6740&#034;\u7684\u6392\u67e5\u90fd\u4ece\u8fd9\u9053\u52a0\u6cd5\u5f00\u59cb\u3002<\/p>\n<\/li>\n<li>\n<p>overhead \u9ed8\u8ba4 heap \u00d7 0.10&#xff08;\u4e0b\u9650 384 MiB&#xff09;\u5bf9\u751f\u4ea7\u51e0\u4e4e\u603b\u662f\u4e0d\u591f\u3002 \u8bfb Parquet \u5411\u91cf\u5316\u3001Python UDF\u3001\u8d85\u5927\u5d4c\u5957\u8bb0\u5f55&#xff0c;\u4e09\u8005\u4efb\u4e00\u5b58\u5728\u5c31\u8981\u663e\u5f0f\u8c03\u5927\u3002\u7528\u9ed8\u8ba4\u503c\u4e0a\u7ebf\u7b49\u4e8e\u8d4c\u8fd0\u6c14\u3002<\/p>\n<\/li>\n<li>\n<p>\u5f00 off-heap \u5fc5\u987b\u540c\u6b65\u8c03\u4f4e\u5806&#xff0c;\u5e76\u91cd\u65b0\u6838\u7b97\u961f\u5217\u914d\u989d\u3002 \u53ea\u52a0 offHeap.size \u4e0d\u6539 executor.memory&#xff0c;\u7b49\u4e8e\u6084\u6084\u62ac\u9ad8\u5bb9\u5668\u9700\u6c42&#xff0c;\u800c\u62a5\u9519\u4fe1\u606f\u91cc\u4e0d\u4f1a\u63d0 off-heap \u4e00\u4e2a\u5b57\u3002<\/p>\n<\/li>\n<li>\n<p>\u9a71\u9010\u662f\u5355\u5411\u7684&#xff1a;\u6267\u884c\u53ef\u9a71\u9010\u5b58\u50a8\u5230 R \u4e3a\u6b62&#xff0c;\u5b58\u50a8\u4e0d\u80fd\u9a71\u9010\u6267\u884c\u3002 \u6240\u4ee5&#034;\u52a0\u4e86 cache \u53cd\u800c\u53d8\u6162&#034;\u8981\u5f80\u4e0b\u8c03 memory.fraction&#xff0c;\u4e0d\u662f\u5f80\u4e0a\u8c03\u3002\u65b9\u5411\u8c03\u53cd\u662f\u8fd9\u7c7b\u95ee\u9898\u7684\u4e3b\u8981\u8017\u65f6\u6765\u6e90\u3002<\/p>\n<\/li>\n<li>\n<p>executor.cores \u589e\u5927\u7b49\u4e8e\u51cf\u5c11\u6bcf\u4e2a task \u7684\u53ef\u7528\u5185\u5b58\u3002 \u5806\u662f executor \u7ea7\u7684&#xff0c;\u4e0d\u968f cores \u5206\u644a\u3002&#034;\u63d0\u9ad8\u5e76\u53d1&#034;\u548c&#034;\u66f4\u591a spill&#034;\u7ecf\u5e38\u662f\u540c\u4e00\u4e2a\u52a8\u4f5c\u3002<\/p>\n<\/li>\n<li>\n<p>\u52a8\u6001\u5206\u914d\u4e09\u5751&#xff1a;minExecutors \u8ba9\u7f29\u5bb9\u5931\u6548\u3001\u6ca1\u6709 shuffle service \u5c31\u4e0d\u80fd\u5f00\u3001\u4e0e AQE \u5408\u5e76\u5206\u533a\u4e92\u76f8\u62b5\u6d88\u3002 \u5224\u65ad\u65b9\u6cd5\u662f\u770b Executors \u9875\u7684 Removal Reason \u5206\u5e03&#xff0c;\u4e0d\u662f\u731c\u3002<\/p>\n<\/li>\n<li>\n<p>\u5bf9\u8c61\u5b58\u50a8\u4e0a&#034;\u6570\u636e\u672c\u5730\u6027&#034;\u4e0d\u6210\u7acb\u3002 \u6e56\u8868\u5728 S3\/OSS \u65f6\u628a locality.wait \u8bbe 0&#xff0c;\u7528 maxPartitionBytes \u5207\u7247\u63a7\u5e76\u884c\u30024.x \u628a maxSinglePartitionBytes \u9ed8\u8ba4\u6539\u6210 128m&#xff0c;\u65b9\u5411\u4e0e\u6b64\u4e00\u81f4\u3002<\/p>\n<\/li>\n<li>\n<p>\u8d44\u6e90\u4e89\u62a2\u7684\u540e\u679c\u662f\u6162&#xff0c;\u5199\u5165\u6743\u51b2\u7a81\u7684\u540e\u679c\u662f\u4e22\u6570\u636e\u3002 \u7b2c\u4e8c\u5c42\u4ef2\u88c1\u6bd4\u7b2c\u4e00\u5c42\u9694\u79bb\u66f4\u91cd\u8981&#xff0c;\u800c\u4e14\u5b83\u5fc5\u987b\u88ab\u5b9e\u6d4b\u2014\u2014\u4e0d\u80fd\u9760 code review \u901a\u8fc7\u3002<\/p>\n<\/li>\n<li>\n<p>\u6279\u4f5c\u4e1a\u7684\u91cd\u8bd5\u662f\u8bbe\u8ba1\u7684\u4e00\u90e8\u5206&#xff0c;\u8fd9\u4f7f Spark \u7684\u5199\u5165\u6743\u51b2\u7a81\u6982\u7387\u8fdc\u9ad8\u4e8e\u5176\u4ed6\u7ec4\u4ef6\u3002 \u4e00\u4e2a\u91cd\u8bd5\u4e09\u6b21\u7684\u56de\u5237\u4f5c\u4e1a\u8986\u76d6\u6d41\u5199\u4f5c\u4e1a\u7684\u5feb\u7167&#xff0c;\u4e24\u8fb9\u90fd\u4e0d\u62a5\u9519\u3002<\/p>\n<\/li>\n<li>\n<p>K8s \u4e0a executor \u53ea\u8bbe requests \u4e0d\u8bbe limits&#xff0c;\u4f1a\u5728\u8282\u70b9\u5185\u5b58\u538b\u529b\u65f6\u5148\u88ab\u9a71\u9010\u3002 YARN \u4e0a\u6ca1\u6709\u5bf9\u5e94\u73b0\u8c61&#xff0c;\u4ece YARN \u8fc1\u5230 K8s \u7684\u56e2\u961f\u4f1a\u628a\u5b83\u8bef\u5224\u6210 Spark \u7684 bug\u3002<\/p>\n<\/li>\n<li>\n<p>\u4e8b\u4ef6\u65e5\u5fd7\u4e0d\u53ef\u8ffd\u6eaf\u751f\u6210&#xff0c;\u5fc5\u987b\u642d\u73af\u5883\u5f53\u5929\u5c31\u5f00\u3002 \u540c\u65f6 logStageExecutorMetrics \u4e5f\u8981\u5f00\u2014\u2014GC \u4e0e spill \u66f2\u7ebf\u662f\u5b9a\u4f4d\u5bb9\u5668\u88ab\u6740\u7684\u552f\u4e00\u8bc1\u636e\u3002<\/p>\n<\/li>\n<li>\n<p>History Server \u5fc5\u987b\u5148\u4e8e\u4f5c\u4e1a\u5347\u7ea7\u3002 \u65e7 server \u8bfb\u4e0d\u4e86\u65b0\u683c\u5f0f\u65e5\u5fd7&#xff0c;\u4e14\u62a5\u9519\u5f62\u6001\u50cf&#034;\u65e5\u5fd7\u635f\u574f&#034;&#xff0c;\u5bb9\u6613\u8bf1\u5bfc\u4eba\u5220\u76ee\u5f55\u3002<\/p>\n<\/li>\n<li>\n<p>4.0 \u8d77\u9ed8\u8ba4 JDK 17 &#043; G1GC&#xff0c;\u4f46 3.x \u9057\u7559\u7684 -XX:&#043;UseParallelGC \u4f1a\u8986\u76d6\u9ed8\u8ba4\u3002 \u7528 jcmd VM.flags \u786e\u8ba4\u5b9e\u9645\u751f\u6548\u7684 GC&#xff0c;\u4e0d\u8981\u770b\u914d\u7f6e\u6587\u4ef6\u3002<\/p>\n<\/li>\n<li>\n<p>cluster \u6a21\u5f0f\u7684 driver UI \u5728\u5bb9\u5668\u5185&#xff08;4040&#xff09;&#xff0c;\u9700\u8981 proxy \u6216 port-forward\u3002 &#034;\u770b\u4e0d\u5230\u5b9e\u65f6 UI&#034;\u4e0d\u662f\u914d\u7f6e\u9519\u8bef\u3002<\/p>\n<\/li>\n<hr \/>\n<h3>\u672c\u7ae0\u5c0f\u7ed3<\/h3>\n<p>\u672c\u7ae0\u89e3\u51b3&#034;\u8dd1\u5728\u54ea\u3001\u7ed9\u591a\u5c11\u3001\u600e\u4e48\u548c Flink \u5171\u5904&#034;\u3002<\/p>\n<ul>\n<li>\u90e8\u7f72\u6a21\u5f0f&#xff08;3.1&#xff09;&#xff1a;\u5224\u65ad\u4f9d\u636e\u53ea\u6709\u4e00\u6761\u2014\u2014Flink \u5728\u54ea Spark \u5c31\u5728\u54ea\u3002\u751f\u4ea7\u6279\u4f5c\u4e1a\u4e00\u5f8b cluster \u6a21\u5f0f&#xff0c;\u56e0\u4e3a driver \u6302\u5728\u63d0\u4ea4\u673a\u4e0a\u7b49\u4e8e\u52a0\u4e86\u4e00\u4e2a&#034;\u4e0d\u80fd\u6389\u7ebf&#034;\u7684\u9690\u542b\u4f9d\u8d56\u3002<\/li>\n<li>\u5185\u5b58\u6a21\u578b&#xff08;3.2&#xff09;&#xff1a;\u5bb9\u5668 &#061; \u5806 &#043; off-heap &#043; overhead \u8fd9\u9053\u52a0\u6cd5&#xff0c;\u662f\u5168\u7ae0\u7684\u652f\u70b9\u3002\u7edf\u4e00\u533a M \u4e0e\u4e0d\u53ef\u9a71\u9010\u533a R \u7684\u5355\u5411\u6027&#xff0c;\u51b3\u5b9a\u4e86&#034;\u52a0\u7f13\u5b58\u53d8\u6162&#034;\u8be5\u5f80\u54ea\u4e2a\u65b9\u5411\u8c03\u3002\u4e09\u79cd\u5bb9\u5668\u88ab\u6740\u7684\u533a\u5206\u8def\u5f84&#xff08;\u7ba1\u7406\u5668\u6740-overhead \/ \u7ba1\u7406\u5668\u6740-offheap \/ JVM \u81ea\u6740&#xff09;\u662f\u672c\u7ae0\u6700\u5b9e\u7528\u7684\u4ea7\u51fa\u3002<\/li>\n<li>\u5e76\u884c\u5ea6&#xff08;3.3&#xff09;&#xff1a;\u4e09\u4e2a\u5c42\u7ea7&#xff08;instances \/ cores \/ partitions&#xff09;\u4e0d\u80fd\u6df7\u4e3a\u4e00\u8c08&#xff1b;AQE \u65f6\u4ee3 shuffle.partitions \u53ef\u4ee5\u8bbe\u5927\u4e0d\u5fc5\u8bbe\u5c0f&#xff1b;\u52a8\u6001\u5206\u914d\u7684\u4e09\u4e2a\u5751\u5404\u6709\u660e\u786e\u5224\u636e\u3002<\/li>\n<li>\u5171\u5b58&#xff08;3.4&#xff09;&#xff1a;\u53cc\u5c42\u9694\u79bb&#xff0c;\u7b2c\u4e8c\u5c42&#xff08;\u5199\u5165\u6743\u4ef2\u88c1&#xff09;\u6bd4\u7b2c\u4e00\u5c42&#xff08;\u8d44\u6e90&#xff09;\u91cd\u8981&#xff0c;\u56e0\u4e3a\u5b83\u7684\u5931\u8d25\u6a21\u5f0f\u662f\u9759\u9ed8\u4e22\u6570\u636e\u3002\u4e09\u5c42\u4ef2\u88c1&#xff08;\u5916\u90e8\u9501 &#043; \u65f6\u95f4\u7a97 &#043; tag \u951a\u5b9a&#xff09;\u5fc5\u987b\u5b9e\u6d4b\u3002<\/li>\n<li>\u53ef\u89c2\u6d4b\u5730\u57fa&#xff08;3.5&#xff09;&#xff1a;\u4e8b\u4ef6\u65e5\u5fd7\u5f53\u5929\u5f00\u3001\u76ee\u5f55\u5206 done\/in-progress\u3001History Server \u5148\u5347\u7ea7\u3002<\/li>\n<\/ul>\n<p>15 \u9879\u4e0a\u7ebf\u6e05\u5355\u91cc&#xff0c;\u7b2c 8 \u9879\u662f\u552f\u4e00&#034;\u540e\u679c\u4e3a\u4e22\u6570\u636e&#034;\u7684\u2014\u2014\u5982\u679c\u65f6\u95f4\u53ea\u591f\u9a8c\u4e00\u9879&#xff0c;\u9a8c\u5b83\u3002<\/p>\n<hr \/>\n<p>\u4e0b\u7ae0\u9884\u544a&#xff1a;\u5e95\u5ea7\u7a33\u4e86&#xff0c;\u5f00\u59cb\u8bb2\u5185\u6838\u3002\u7b2c 4 \u7ae0\u300aCatalyst\u3001\u6267\u884c\u8ba1\u5212\u4e0e AQE\u300b\u2014\u2014\u4e00\u6761 SQL \u4ece\u6587\u672c\u5230\u7269\u7406\u8ba1\u5212\u8981\u8fc7\u54ea\u56db\u6bb5\u53d8\u6362\u3001Whole-Stage CodeGen \u4e3a\u4ec0\u4e48\u80fd\u628a\u9010\u884c\u5904\u7406\u7f16\u8bd1\u6210\u4e00\u6bb5 Java\u3001EXPLAIN \u7684\u4e94\u79cd\u8f93\u51fa\u5404\u81ea\u56de\u7b54\u4ec0\u4e48\u95ee\u9898\u3001AQE \u7684\u56db\u4e2a\u5f00\u5173\u600e\u4e48\u786e\u8ba4&#034;\u771f\u7684\u751f\u6548\u4e86&#034;\u800c\u4e0d\u662f&#034;\u914d\u4e86\u4f46\u6ca1\u7528&#034;&#xff0c;\u4ee5\u53ca\u4e00\u5957\u8de8\u5f15\u64ce\u7684\u8bfb\u8ba1\u5212\u65b9\u6cd5&#xff1a;Spark \u7684 EXPLAIN\u3001ClickHouse \u7684 EXPLAIN indexes &#061; 1\u3001Doris \u7684 EXPLAIN&#xff0c;\u4e09\u5957\u5730\u56fe\u600e\u4e48\u5bf9\u7167\u7740\u770b\u3002\u5b66\u5b8c\u8fd9\u7ae0&#xff0c;\u4f60\u62ff\u5230\u4e00\u6761 SQL \u5e94\u8be5\u80fd\u5728\u4e09\u5206\u949f\u5185\u5224\u65ad\u5b83\u4f1a\u4e0d\u4f1a\u6162\u3001\u6162\u5728\u54ea\u3002<\/p>\n<hr \/>\n<h3>\u5b98\u65b9\u53c2\u8003\u8d44\u6599<\/h3>\n<table>\n<tr>\u4e3b\u9898\u94fe\u63a5<\/tr>\n<tbody>\n<tr>\n<td>Tuning Spark&#xff08;\u5185\u5b58\u6a21\u578b\u3001M\/R\u3001GC \u8c03\u4f18\u3001\u5e76\u884c\u5ea6\u3001\u672c\u5730\u6027&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/tuning.html<\/td>\n<\/tr>\n<tr>\n<td>Spark Configuration&#xff08;executor\/driver \u8d44\u6e90\u3001\u52a8\u6001\u5206\u914d\u3001\u5185\u5b58\u53c2\u6570&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/configuration.html<\/td>\n<\/tr>\n<tr>\n<td>Running Spark on YARN<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/running-on-yarn.html<\/td>\n<\/tr>\n<tr>\n<td>Running Spark on Kubernetes<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/running-on-kubernetes.html<\/td>\n<\/tr>\n<tr>\n<td>Spark Job Monitoring&#xff08;Web UI \/ \u4e8b\u4ef6\u65e5\u5fd7 \/ History Server&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/job-monitoring.html<\/td>\n<\/tr>\n<tr>\n<td>Spark SQL Performance Tuning&#xff08;AQE\u3001join \u7b56\u7565\u3001\u5206\u533a\u6570&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/sql-performance-tuning.html<\/td>\n<\/tr>\n<tr>\n<td>Spark 4.0.2 YARN \u6587\u6863&#xff08;AM memoryOverhead &#061; 0.10 \/ min 384&#xff09;<\/td>\n<td>https:\/\/spark.incubator.apache.org\/docs\/4.0.2\/running-on-yarn.html<\/td>\n<\/tr>\n<tr>\n<td>Spark 4.1.0 Configuration<\/td>\n<td>https:\/\/spark.apache.org\/docs\/4.1.0\/configuration.html<\/td>\n<\/tr>\n<tr>\n<td>Iceberg \u5b98\u65b9\u591a\u5f15\u64ce\u652f\u6301&#xff08;\u5e76\u53d1\u63d0\u4ea4\u4e0e\u9501&#xff09;<\/td>\n<td>https:\/\/iceberg.apache.org\/multi-engine-support\/<\/td>\n<\/tr>\n<tr>\n<td>Iceberg Spark \u914d\u7f6e&#xff08;catalog \u4e0e\u9501\u76f8\u5173\u53c2\u6570&#xff09;<\/td>\n<td>https:\/\/iceberg.apache.org\/docs\/latest\/spark-configuration\/<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>\u7248\u672c\u53d6\u8bc1\u8bf4\u660e<\/h3>\n<table>\n<tr>\u5185\u5bb9\u6863\u4f4d\u8bf4\u660e<\/tr>\n<tbody>\n<tr>\n<td>spark.memory.fraction &#061; (JVM heap \u2212 300MiB) \u7684\u6bd4\u4f8b&#xff0c;\u9ed8\u8ba4 0.6&#xff1b;\u5176\u4f59 40% \u7559\u7ed9\u7528\u6237\u6570\u636e\u7ed3\u6784\u4e0e OOM \u7f13\u51b2<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>Tuning \u9875 Memory Management Overview<\/td>\n<\/tr>\n<tr>\n<td>spark.memory.storageFraction &#061; R \u5360 M \u7684\u6bd4\u4f8b&#xff0c;\u9ed8\u8ba4 0.5&#xff1b;R \u5185\u7f13\u5b58\u5757\u4e0d\u88ab\u6267\u884c\u9a71\u9010<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>\u540c\u4e0a<\/td>\n<\/tr>\n<tr>\n<td>\u6267\u884c\u4e0e\u5b58\u50a8\u5171\u4eab\u7edf\u4e00\u533a M&#xff1b;\u6267\u884c\u53ef\u9a71\u9010\u5b58\u50a8\u81f3 R&#xff0c;\u5b58\u50a8\u4e0d\u53ef\u9a71\u9010\u6267\u884c&#xff08;\u5b98\u65b9\u7406\u7531&#xff1a;\u5b9e\u73b0\u590d\u6742\u5ea6&#xff09;<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>\u540c\u4e0a<\/td>\n<\/tr>\n<tr>\n<td>\u201ccache fewer objects than slow down task execution\u201d&#xff08;OldGen \u63a5\u8fd1\u6ee1\u65f6\u964d\u4f4e fraction&#xff09;<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>Tuning \u9875 GC Tuning<\/td>\n<\/tr>\n<tr>\n<td>full GC \u5728 task \u5b8c\u6210\u524d\u591a\u6b21\u89e6\u53d1 &#061; \u6267\u884c\u5185\u5b58\u4e0d\u8db3<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>Tuning \u9875 GC Tuning<\/td>\n<\/tr>\n<tr>\n<td>Since 4.0.0, Spark uses JDK 17 by default, which also makes G1GC the default&#xff1b;\u5927\u5806\u9700\u8c03 -XX:G1HeapRegionSize<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>Tuning \u9875 GC Tuning<\/td>\n<\/tr>\n<tr>\n<td>YARN AM memoryOverhead \u9ed8\u8ba4 &#061; AM memory \u00d7 0.10&#xff0c;\u6700\u5c0f 384 MiB<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>Spark 4.0.2 running-on-yarn \u9875<\/td>\n<\/tr>\n<tr>\n<td>G1 humongous \u5224\u5b9a\u9608\u503c\u4e3a region \u7684\u4e00\u534a\u3001region \u6309\u5806\u5bb9\u91cf\u63a8\u7b97<\/td>\n<td>\u901a\u884c\u7ea6\u5b9a<\/td>\n<td>JVM \u901a\u7528\u884c\u4e3a&#xff0c;\u975e Spark \u6587\u6863&#xff1b;\u6b63\u6587\u4ee5&#034;\u901a\u5e38&#034;\u8868\u8ff0<\/td>\n<\/tr>\n<tr>\n<td>spark.executor.memoryOverhead \u9ed8\u8ba4 &#061; heap \u00d7 0.10 \u4e14\u4e0b\u9650 384 MiB<\/td>\n<td>\u5b98\u65b9\u539f\u6587&#xff08;\u914d\u7f6e\u9875\u957f\u671f\u7a33\u5b9a\u503c&#xff09;<\/td>\n<td>\u672c\u6b21\u7ecf YARN \u9875 AM \u540c\u6784\u53c2\u6570\u65c1\u8bc1&#xff1b;\u9010\u5b57\u590d\u6838\u89c1\u4e0b\u65b9\u5f85\u529e<\/td>\n<\/tr>\n<tr>\n<td>spark.dynamicAllocation.minExecutors \u9ed8\u8ba4 &#061; initialExecutors<\/td>\n<td>\u5b98\u65b9\u539f\u6587&#xff08;\u914d\u7f6e\u9875&#xff09;<\/td>\n<td>\u672c\u6b21\u672a\u9010\u5b57\u6293\u53d6 configuration.html&#xff0c;\u5f15\u7528\u65f6\u6807\u6ce8\u4e3a\u914d\u7f6e\u9875\u53ef\u67e5<\/td>\n<\/tr>\n<tr>\n<td>executorIdleTimeout \u9ed8\u8ba4 60s\u3001cachedExecutorIdleTimeout \u9ed8\u8ba4 5min\u3001shuffleTracking \u4e0e shuffle.service \u4e92\u65a5<\/td>\n<td>\u5b98\u65b9\u539f\u6587&#xff08;\u914d\u7f6e\u9875&#xff09;<\/td>\n<td>\u540c\u4e0a&#xff0c;\u52a8\u7b14\u7b2c 17 \u7ae0\u524d\u9010\u5b57\u590d\u6838<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.shuffle.partitions \u9ed8\u8ba4 200\u3001spark.executor.cores \u9ed8\u8ba4 1\u3001spark.locality.wait \u9ed8\u8ba4 3s<\/td>\n<td>\u5b98\u65b9\u539f\u6587&#xff08;\u914d\u7f6e\u9875&#xff09;<\/td>\n<td>\u540c\u4e0a<\/td>\n<\/tr>\n<tr>\n<td>AQE \u9ed8\u8ba4\u5f00\u542f\u3001advisoryPartitionSizeInBytes \u9ed8\u8ba4 64MB\u3001minPartitionNum \u9ed8\u8ba4 1<\/td>\n<td>\u5b98\u65b9\u539f\u6587&#xff08;SQL \u6027\u80fd\u8c03\u4f18\u9875&#xff09;<\/td>\n<td>\u672c\u6b21\u672a\u9010\u5b57\u6293\u53d6&#xff0c;\u5f15\u7528\u65f6\u6807\u6ce8<\/td>\n<\/tr>\n<tr>\n<td>History Server \u53ef\u8bfb\u65e7\u7248\u672c\u65e5\u5fd7\u3001\u65e7 server \u4e0d\u53ef\u8bfb\u65b0\u683c\u5f0f<\/td>\n<td>\u901a\u884c\u7ea6\u5b9a<\/td>\n<td>\u5b98\u65b9\u672a\u4ee5&#034;\u517c\u5bb9\u6027\u8868&#034;\u5f62\u5f0f\u660e\u786e&#xff1b;\u6b63\u6587\u4ee5&#034;\u5b9e\u6d4b\u786e\u8ba4&#034;\u8981\u6c42\u8bfb\u8005\u81ea\u67e5<\/td>\n<\/tr>\n<tr>\n<td>\u8868 2.5.1 \u4e2d spark.sql.maxSinglePartitionBytes \u9ed8\u8ba4\u6539\u4e3a 128m&#xff08;4.0 \u8d77&#xff09;<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>SQL \u8fc1\u79fb\u6307\u5357&#xff08;\u7b2c 2 \u7ae0\u5df2\u9010\u5b57\u53d6\u8bc1&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>3.4.2 \u53cc\u5199\u51b2\u7a81\u6d4b\u8bd5\u7684\u5fc5\u8981\u6027\u3001\u4e09\u5c42\u4ef2\u88c1\u65b9\u6848<\/td>\n<td>\u901a\u884c\u7ea6\u5b9a &#043; \u524d\u4f5c\u786c\u7ea6\u675f<\/td>\n<td>\u524d\u4f5c Flink &#043; Kafka Connect \u7ed3\u8bba\u7684\u76f4\u63a5\u63a8\u5e7f<\/td>\n<\/tr>\n<tr>\n<td>\u961f\u5217 maximum-capacity \u7559\u4f59\u91cf\u3001K8s requests\/limits \u4e0e\u9a71\u9010<\/td>\n<td>\u901a\u884c\u7ea6\u5b9a<\/td>\n<td>\u8c03\u5ea6\u5668 \/ K8s \u901a\u7528\u5de5\u7a0b\u5b9e\u8df5<\/td>\n<\/tr>\n<tr>\n<td>3.6 \u4e0a\u7ebf checklist 15 \u9879<\/td>\n<td>\u901a\u884c\u7ea6\u5b9a<\/td>\n<td>\u5de5\u7a0b\u7ecf\u9a8c\u5f52\u7eb3&#xff0c;\u975e\u5b98\u65b9\u89c4\u8303<\/td>\n<\/tr>\n<tr>\n<td>\u5404\u7c7b\u62a5\u9519\u539f\u6587&#xff08;Exit code 137\u3001Container killed by YARN\u3001CommitFailedException&#xff09;<\/td>\n<td>\u7b14\u8005\u793a\u610f&#xff08;\u5f62\u6001\u771f\u5b9e&#xff09;<\/td>\n<td>\u62a5\u9519\u6587\u672c\u5f62\u6001\u53d6\u81ea\u5e38\u89c1\u771f\u5b9e\u65e5\u5fd7&#xff0c;\u5177\u4f53\u63aa\u8f9e\u968f\u53d1\u884c\u7248\u4e0e\u7248\u672c\u53d8\u5316<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u672c\u7ae0\u9057\u7559\u5f85\u529e&#xff1a;configuration.html \u4e0e sql-performance-tuning.html \u7684\u9ed8\u8ba4\u503c\u672c\u6b21\u672a\u9010\u5b57\u6293\u53d6&#xff0c;\u4e0a\u8868\u5df2\u6807\u6ce8\u3002\u7b2c 17 \u7ae0&#xff08;\u6027\u80fd\u8c03\u4f18&#xff09;\u52a8\u7b14\u524d\u5fc5\u987b\u8865\u9f50\u9010\u5b57\u590d\u6838\u2014\u2014\u90a3\u4e00\u7ae0\u4f1a\u5927\u91cf\u5f15\u7528\u8fd9\u4e9b\u9ed8\u8ba4\u503c\u3002<\/p>\n<hr \/>\n<h3>\u5b9e\u64cd\u6e05\u5355<\/h3>\n<p>\u672c\u7ae0\u4e3a\u90e8\u7f72\u7ae0&#xff0c;\u4ee5\u4e0b\u6bcf\u9879\u53ef\u590d\u73b0\u3002\u7b2c 4\u30018 \u9879\u662f\u672c\u7ae0\u7684\u6838\u5fc3\u5b9e\u9a8c\u3002<\/p>\n<table>\n<tr>#\u64cd\u4f5c\u547d\u4ee4 \/ \u4f4d\u7f6e\u9a8c\u6536\u6807\u51c6<\/tr>\n<tbody>\n<tr>\n<td>1<\/td>\n<td>YARN cluster \u6a21\u5f0f\u63d0\u4ea4<\/td>\n<td>3.1.2 \u914d\u7f6e\u5757<\/td>\n<td>RM UI \u770b\u5230 ACCEPTED\u2192RUNNING\u2192FINISHED&#xff1b;AM \u5728\u96c6\u7fa4\u5185<\/td>\n<\/tr>\n<tr>\n<td>2<\/td>\n<td>K8s \u6a21\u5f0f\u63d0\u4ea4<\/td>\n<td>3.1.3 \u547d\u4ee4<\/td>\n<td>driver pod \u5148\u8d77&#xff0c;executor \u9010\u4e2a\u51fa\u73b0&#xff0c;\u7ed3\u675f\u540e\u88ab\u6e05\u7406<\/td>\n<\/tr>\n<tr>\n<td>3<\/td>\n<td>\u624b\u7b97\u5bb9\u5668\u603b\u989d<\/td>\n<td>heap &#043; offHeap &#043; overhead<\/td>\n<td>\u4e0e yarn application -status \u6216 pod requests \u4e00\u81f4<\/td>\n<\/tr>\n<tr>\n<td>4<\/td>\n<td>\u590d\u73b0\u5bb9\u5668\u88ab\u6740<\/td>\n<td>\u6545\u610f\u628a memoryOverhead \u8bbe 384m \u8dd1 Parquet \u5411\u91cf\u5316\u8bfb\u53d6<\/td>\n<td>\u65e5\u5fd7\u51fa\u73b0 exceeding memory limits&#xff1b;\u8c03\u5927\u5230 2g \u540e\u6d88\u5931<\/td>\n<\/tr>\n<tr>\n<td>5<\/td>\n<td>\u9a8c\u8bc1 M\/R \u5355\u5411\u9a71\u9010<\/td>\n<td>\u8bbe memory.fraction&#061;0.9 &#043; \u5927\u91cf cache() &#043; \u805a\u5408<\/td>\n<td>\u51fa\u73b0 spill&#xff1b;\u964d\u56de 0.6 \u540e spill \u6d88\u5931<\/td>\n<\/tr>\n<tr>\n<td>6<\/td>\n<td>\u786e\u8ba4\u5b9e\u9645\u751f\u6548\u7684 GC<\/td>\n<td>jcmd &lt;pid&gt; VM.flags | grep -oE &#039;Use[A-Za-z]*GC&#039;<\/td>\n<td>\u8f93\u51fa G1GC&#xff1b;\u82e5\u4e3a Parallel \u5219\u6e05\u7406\u9057\u7559\u53c2\u6570<\/td>\n<\/tr>\n<tr>\n<td>7<\/td>\n<td>\u89c2\u5bdf AQE \u5408\u5e76\u6548\u679c<\/td>\n<td>SQL tab \u770b\u6700\u7ec8 stage task \u6570<\/td>\n<td>\u660e\u663e\u5c0f\u4e8e shuffle.partitions \u5373\u751f\u6548<\/td>\n<\/tr>\n<tr>\n<td>8<\/td>\n<td>\u53cc\u5199\u51b2\u7a81\u6d4b\u8bd5<\/td>\n<td>3.4.2 \u4e09\u6b65\u6d41\u7a0b<\/td>\n<td>\u540e\u5230\u8005\u629b CommitFailedException&#xff1b;\u82e5\u4e24\u8fb9\u90fd&#034;\u6210\u529f&#034;\u5219\u9501\u672a\u751f\u6548&#xff0c;\u7981\u6b62\u4e0a\u751f\u4ea7<\/td>\n<\/tr>\n<tr>\n<td>9<\/td>\n<td>\u9a8c\u8bc1\u52a8\u6001\u5206\u914d\u771f\u7684\u7f29\u5bb9<\/td>\n<td>Executors \u9875\u770b Removal Reason<\/td>\n<td>\u51fa\u73b0 idle \u4e14\u6570\u91cf\u968f\u8d1f\u8f7d\u4e0b\u964d&#xff1b;\u82e5\u59cb\u7ec8\u4e3a 0 \u5219 minExecutors \u8bbe\u9519<\/td>\n<\/tr>\n<tr>\n<td>10<\/td>\n<td>\u4e8b\u4ef6\u65e5\u5fd7\u751f\u6210\u786e\u8ba4<\/td>\n<td>hdfs dfs -ls \/spark-history\/in-progress<\/td>\n<td>\u8fd0\u884c\u4e2d\u6709\u6587\u4ef6\u4e14\u6301\u7eed\u589e\u957f<\/td>\n<\/tr>\n<tr>\n<td>11<\/td>\n<td>\u5f52\u6863\u4e0e History \u53ef\u89c1<\/td>\n<td>&#8230;\/done &#043; curl :18080\/json\/version<\/td>\n<td>\u4f5c\u4e1a\u7ed3\u675f\u540e\u6587\u4ef6\u5728 done\/&#xff0c;UI \u53ef\u6253\u5f00<\/td>\n<\/tr>\n<tr>\n<td>12<\/td>\n<td>\u672c\u5730\u6027\u5728\u5bf9\u8c61\u5b58\u50a8\u4e0a\u5f52\u96f6<\/td>\n<td>\u8bbe locality.wait&#061;0s \u524d\u540e\u5404\u8dd1\u4e00\u6b21<\/td>\n<td>\u8017\u65f6\u4e0d\u589e&#xff08;\u82e5\u660e\u663e\u53d8\u6162\u8bf4\u660e\u6570\u636e\u5176\u5b9e\u5728 HDFS&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>13<\/td>\n<td>\u961f\u5217\u914d\u989d\u7559\u4f59\u91cf\u6838\u7b97<\/td>\n<td>\u6bd4\u5bf9\u6279\u961f\u5217 maximum-capacity \u4e0e\u6d41\u961f\u5217 capacity<\/td>\n<td>\u4e8c\u8005\u4e4b\u548c \u2264 100 \u4e14\u6d41\u4fa7\u6709\u4f59\u91cf<\/td>\n<\/tr>\n<tr>\n<td>14<\/td>\n<td>\u5e42\u7b49\u91cd\u8dd1\u9a8c\u8bc1<\/td>\n<td>\u624b\u52a8 kill \u4f5c\u4e1a\u8ba9\u8c03\u5ea6\u5668\u91cd\u8dd1<\/td>\n<td>\u7ed3\u679c\u884c\u6570\u4e0e\u9996\u6b21\u4e00\u81f4&#xff0c;\u65e0\u91cd\u590d<\/td>\n<\/tr>\n<tr>\n<td>15<\/td>\n<td>\u56de\u6eda\u6f14\u7ec3<\/td>\n<td>\u7528 tag \u6062\u590d\u5230\u63a5\u7ba1\u524d\u5feb\u7167<\/td>\n<td>\u8868\u5185\u5bb9\u4e0e tag \u65f6\u523b\u4e00\u81f4&#xff0c;Flink \u6062\u590d\u76f4\u5199<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>\u601d\u8003\u9898<\/h3>\n<li>\n<p>\u67d0\u4f5c\u4e1a\u914d\u7f6e spark.executor.memory&#061;8g\u3001spark.executor.cores&#061;8\u3001spark.sql.shuffle.partitions&#061;200&#xff0c;\u961f\u5217\u7ed9\u6bcf\u5bb9\u5668 9g\u3002\u5de5\u7a0b\u5e08\u62a5\u544a&#034;\u5185\u5b58\u4e0d\u591f&#xff0c;\u7533\u8bf7\u52a0\u5230 16g&#034;\u3002\u8bf7\u6309\u672c\u7ae0\u7684\u7b97\u6cd5\u5224\u65ad&#xff1a;\u771f\u6b63\u4e0d\u591f\u7684\u662f\u4ec0\u4e48&#xff1f;\u52a0\u5806\u662f\u4e0d\u662f\u6b63\u786e\u89e3\u6cd5&#xff1f;<\/p>\n<\/li>\n<li>\n<p>3.4.2 \u7684\u4e09\u5c42\u4ef2\u88c1\u91cc&#xff0c;\u201c\u65f6\u95f4\u7a97&#034;\u8fd9\u4e00\u5c42\u4f9d\u8d56\u8c03\u5ea6\u5668\u800c\u4e0d\u662f\u4f5c\u4e1a\u81ea\u8eab\u3002\u8bf7\u8bbe\u8ba1\u4e00\u4e2a\u53cd\u4f8b&#xff1a;\u4ec0\u4e48\u60c5\u51b5\u4e0b\u4f5c\u4e1a\u4f1a\u8d85\u65f6\u8de8\u51fa\u7a97\u53e3&#xff0c;\u6b64\u65f6\u4f1a\u53d1\u751f\u4ec0\u4e48&#xff0c;\u4f60\u600e\u4e48\u7528\u6280\u672f\u624b\u6bb5&#xff08;\u4e0d\u662f&#034;\u628a\u7a97\u53e3\u8bbe\u5927\u4e00\u70b9\u201d&#xff09;\u5835\u4f4f\u5b83&#xff1f;<\/p>\n<\/li>\n<li>\n<p>\u5b98\u65b9\u8bf4&#034;\u5b58\u50a8\u4e0d\u80fd\u9a71\u9010\u6267\u884c&#034;\u3002\u8fd9\u4e2a\u4e0d\u5bf9\u79f0\u6027\u5bf9\u6279\u4f5c\u4e1a\u548cStructured Streaming \u4f5c\u4e1a\u5206\u522b\u610f\u5473\u7740\u4ec0\u4e48&#xff1f;\u63d0\u793a&#xff1a;\u6d41\u4f5c\u4e1a\u7684\u72b6\u6001\u5c5e\u4e8e storage \u8fd8\u662f execution&#xff1f;&#xff08;\u9700\u67e5\u8bc1\u540e\u56de\u7b54&#xff0c;\u672c\u7ae0\u672a\u8986\u76d6&#xff09;<\/p>\n<\/li>\n<li>\n<p>3.3.3 \u6613\u9519\u70b9 3 \u63cf\u8ff0\u4e86&#034;\u52a8\u6001\u5206\u914d\u4e0e AQE \u5408\u5e76\u4e92\u76f8\u62b5\u6d88&#034;\u7684\u6296\u52a8\u3002\u5982\u679c\u96c6\u7fa4\u4e0a\u540c\u65f6\u8dd1 50 \u4e2a\u6279\u4f5c\u4e1a&#xff0c;\u6bcf\u4e2a\u90fd\u6709\u8fd9\u79cd\u6296\u52a8&#xff0c;\u4f1a\u5f62\u6210\u4ec0\u4e48\u7cfb\u7edf\u7ea7\u73b0\u8c61&#xff1f;\u4f60\u4f1a\u5728\u8c03\u5ea6\u5668\u5c42\u9762\u8fd8\u662f Spark \u914d\u7f6e\u5c42\u9762\u89e3\u51b3&#xff1f;<\/p>\n<\/li>\n<li>\n<p>\u4e0a\u7ebf\u6e05\u5355\u7b2c 8 \u9879&#xff08;\u5199\u5165\u6743\u4ef2\u88c1&#xff09;\u7684\u540e\u679c\u662f&#034;\u9759\u9ed8\u4e22\u6570\u636e&#034;&#xff0c;\u7b2c 4\u30017 \u9879\u7684\u540e\u679c\u662f&#034;\u4f5c\u4e1a\u5931\u8d25\/\u53d8\u6162&#034;\u3002\u4e3a\u4ec0\u4e48\u4eba\u4eec\u901a\u5e38\u628a\u529b\u6c14\u82b1\u5728\u540e\u8005\u4e0a&#xff1f;\u7ed9\u51fa\u4e00\u4e2a\u80fd\u8ba9\u7b2c 8 \u9879\u5728\u8bc4\u5ba1\u4f1a\u4e0a\u771f\u6b63\u88ab\u91cd\u89c6\u7684\u5177\u4f53\u505a\u6cd5\u3002<\/p>\n<\/li>\n<hr \/>\n<p>\u4e0b\u4e00\u7ae0\u8fdb\u5165\u5185\u6838\u3002\u524d\u4e09\u7ae0\u89e3\u51b3\u4e86&#034;\u8be5\u4e0d\u8be5\u7528\u3001\u8dd1\u5728\u54ea\u3001\u7ed9\u591a\u5c11&#034;&#xff0c;\u4ece\u8fd9\u91cc\u5f00\u59cb\u56de\u7b54\u90a3\u4e2a\u66f4\u672c\u8d28\u7684\u95ee\u9898&#xff1a;\u4e00\u6761 SQL \u4ea4\u7ed9 Spark&#xff0c;\u5b83\u5230\u5e95\u8bfb\u4e86\u591a\u5c11\u6570\u636e\u3001\u4e3a\u4ec0\u4e48\u662f\u8fd9\u4e2a\u6570\u3001\u4ee5\u53ca\u600e\u4e48\u8ba9\u5b83\u5c11\u8bfb\u5341\u500d\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\/* MD \/ \u5bcc\u6587\u672c\u4e2d\u7684 .toc\uff08\u542b\u535a\u5ba2\u56ed\u642c\u5bb6\u7b49\u5d4c\u5957\u7ed3\u6784\uff09\uff1b.toc-box \u5728\u4fa7\u680f\uff0c\u4e0d\u53d7\u5f71\u54cd *\/#content_views .toc,\/* \u7f16\u8f91\u5668\u5e38\u5728\u76ee\u5f55\u524d\u540e\u63d2\u5165\u7a7a p\uff08:empty \u4ecd\u5360 20px\uff09\uff0c\u4e00\u5e76\u53bb\u6389\u907f\u514d\u9876\u7a7a\u9699 *\/#content_views.markdown_views &gt; p:empty:has(+ .toc),#content_views.markdown_views &gt; .toc + p:empty,\/* \u5bcc\u6587\u672c\u65e7\u7248\u76ee\u5f55\u6807\u8bb0 *\/#content<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[365,664,323],"topic":[],"class_list":["post-123506","post","type-post","status-publish","format-standard","hentry","category-server","tag-spark","tag-664","tag-323"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u7b2c3\u7ae0 Spark4.x \u90e8\u7f72\u6a21\u5f0f\u3001\u8d44\u6e90\u6a21\u578b\u4e0e\u5b58\u91cf\u8fc1\u79fb - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/123506.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u7b2c3\u7ae0 Spark4.x \u90e8\u7f72\u6a21\u5f0f\u3001\u8d44\u6e90\u6a21\u578b\u4e0e\u5b58\u91cf\u8fc1\u79fb - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\/* MD \/ \u5bcc\u6587\u672c\u4e2d\u7684 .toc\uff08\u542b\u535a\u5ba2\u56ed\u642c\u5bb6\u7b49\u5d4c\u5957\u7ed3\u6784\uff09\uff1b.toc-box \u5728\u4fa7\u680f\uff0c\u4e0d\u53d7\u5f71\u54cd *\/#content_views .toc,\/* \u7f16\u8f91\u5668\u5e38\u5728\u76ee\u5f55\u524d\u540e\u63d2\u5165\u7a7a p\uff08:empty \u4ecd\u5360 20px\uff09\uff0c\u4e00\u5e76\u53bb\u6389\u907f\u514d\u9876\u7a7a\u9699 *\/#content_views.markdown_views &gt; p:empty:has(+ .toc),#content_views.markdown_views &gt; .toc + p:empty,\/* \u5bcc\u6587\u672c\u65e7\u7248\u76ee\u5f55\u6807\u8bb0 *\/#content\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/123506.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-11-07T07:25:04+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"16 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/123506.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/123506.html\",\"name\":\"\u7b2c3\u7ae0 Spark4.x \u90e8\u7f72\u6a21\u5f0f\u3001\u8d44\u6e90\u6a21\u578b\u4e0e\u5b58\u91cf\u8fc1\u79fb - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-11-07T07:25:04+00:00\",\"dateModified\":\"2026-11-07T07:25:04+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/123506.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/123506.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/123506.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u7b2c3\u7ae0 Spark4.x \u90e8\u7f72\u6a21\u5f0f\u3001\u8d44\u6e90\u6a21\u578b\u4e0e\u5b58\u91cf\u8fc1\u79fb\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u7b2c3\u7ae0 Spark4.x \u90e8\u7f72\u6a21\u5f0f\u3001\u8d44\u6e90\u6a21\u578b\u4e0e\u5b58\u91cf\u8fc1\u79fb - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/123506.html","og_locale":"zh_CN","og_type":"article","og_title":"\u7b2c3\u7ae0 Spark4.x \u90e8\u7f72\u6a21\u5f0f\u3001\u8d44\u6e90\u6a21\u578b\u4e0e\u5b58\u91cf\u8fc1\u79fb - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\/* MD \/ \u5bcc\u6587\u672c\u4e2d\u7684 .toc\uff08\u542b\u535a\u5ba2\u56ed\u642c\u5bb6\u7b49\u5d4c\u5957\u7ed3\u6784\uff09\uff1b.toc-box \u5728\u4fa7\u680f\uff0c\u4e0d\u53d7\u5f71\u54cd *\/#content_views .toc,\/* \u7f16\u8f91\u5668\u5e38\u5728\u76ee\u5f55\u524d\u540e\u63d2\u5165\u7a7a p\uff08:empty \u4ecd\u5360 20px\uff09\uff0c\u4e00\u5e76\u53bb\u6389\u907f\u514d\u9876\u7a7a\u9699 *\/#content_views.markdown_views &gt; p:empty:has(+ .toc),#content_views.markdown_views &gt; .toc + p:empty,\/* \u5bcc\u6587\u672c\u65e7\u7248\u76ee\u5f55\u6807\u8bb0 *\/#content","og_url":"https:\/\/www.wsisp.com\/helps\/123506.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-11-07T07:25:04+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"16 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/123506.html","url":"https:\/\/www.wsisp.com\/helps\/123506.html","name":"\u7b2c3\u7ae0 Spark4.x \u90e8\u7f72\u6a21\u5f0f\u3001\u8d44\u6e90\u6a21\u578b\u4e0e\u5b58\u91cf\u8fc1\u79fb - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-11-07T07:25:04+00:00","dateModified":"2026-11-07T07:25:04+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/123506.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/123506.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/123506.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u7b2c3\u7ae0 Spark4.x \u90e8\u7f72\u6a21\u5f0f\u3001\u8d44\u6e90\u6a21\u578b\u4e0e\u5b58\u91cf\u8fc1\u79fb"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/123506","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=123506"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/123506\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=123506"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=123506"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=123506"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=123506"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}