{"id":124025,"date":"2026-11-08T19:40:55","date_gmt":"2026-11-08T11:40:55","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/124025.html"},"modified":"2026-11-08T19:40:55","modified_gmt":"2026-11-08T11:40:55","slug":"%e7%ac%ac5%e7%ab%a0-spark4-x-%e5%88%86%e5%8c%ba%e3%80%81%e5%88%86%e6%a1%b6%e3%80%81%e6%8e%92%e5%ba%8f%e4%b8%8e%e6%95%b0%e6%8d%ae%e8%a3%81%e5%89%aa","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/124025.html","title":{"rendered":"\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a"},"content":{"rendered":"<h2>\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a<\/h2>\n<p>\u7ae0\u8282\u5b9a\u4f4d&#xff1a;\u7b2c 4 \u7ae0\u6559\u4f60\u8bfb\u8ba1\u5212&#xff0c;\u672c\u7ae0\u6559\u4f60\u8ba9\u8ba1\u5212\u6709\u4e1c\u897f\u53ef\u88c1\u3002\u4f18\u5316\u5668\u518d\u806a\u660e&#xff0c;\u4e5f\u53ea\u80fd\u5728\u7269\u7406\u5e03\u5c40\u7ed9\u51fa\u7684\u53ef\u80fd\u6027\u91cc\u6311\u3002\u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u8fd9\u4e09\u4ef6\u4e8b\u662f\u5efa\u8868\u65f6\u4e00\u6b21\u5b9a\u6b7b\u3001\u4e8b\u540e\u6539\u8d77\u6765\u6700\u8d35\u7684\u4e09\u4e2a\u51b3\u5b9a&#xff0c;\u6240\u4ee5\u5fc5\u987b\u653e\u5728\u6e56\u4ed3\u8bfb\u5199&#xff08;\u7b2c 8-11 \u7ae0&#xff09;\u4e4b\u524d\u8bb2\u900f\u3002<\/p>\n<p>\u672c\u7ae0\u540c\u65f6\u662f\u5168\u7cfb\u5217\u4e0e\u524d\u4f5c\u7684\u4e00\u6b21\u6b63\u5f0f\u5bf9\u9f50&#xff1a;ck \u7b2c 4 \u7ae0\u90a3\u5f20&#034;\u5206\u533a \u2192 \u4e3b\u952e \u2192 \u8df3\u6570\u7d22\u5f15 \u2192 \u9884\u8ba1\u7b97&#034;\u7684\u88c1\u526a\u6f0f\u6597&#xff0c;\u5728 Spark &#043; \u5217\u5f0f\u6587\u4ef6\u4e0a\u7684\u5bf9\u5e94\u5f62\u6001\u662f&#034;\u5206\u533a \u2192 \u6587\u4ef6 \u2192 row-group \u2192 \u884c&#034;\u3002\u56db\u5c42\u987a\u5e8f\u4e0d\u53d8&#xff0c;\u6bcf\u5c42\u7684\u540d\u5b57\u548c\u89c2\u6d4b\u6307\u6807\u4e0d\u540c&#xff0c;\u5224\u636e\u5b8c\u5168\u4e00\u81f4&#xff1a;\u5148\u95ee\u8fd9\u4e00\u5c42\u88c1\u6389\u4e86\u591a\u5c11&#xff0c;\u518d\u95ee\u4e0b\u4e00\u5c42\u503c\u4e0d\u503c\u5f97\u505a\u3002<\/p>\n<hr \/>\n<h3>\u5b66\u4e60\u76ee\u6807<\/h3>\n<ul>\n<li>\u2713 \u5206\u5f97\u6e05\u4e09\u79cd&#034;\u5206\u533a&#034;&#xff08;Hive \u76ee\u5f55\u5206\u533a \/ Iceberg \u9690\u85cf\u5206\u533a \/ Paimon \u5206\u533a &#043; Bucket&#xff09;&#xff0c;\u80fd\u5404\u81ea\u8bf4\u51fa&#xff1a;\u88c1\u526a\u7531\u8c01\u8d1f\u8d23\u3001\u8c13\u8bcd\u600e\u4e48\u5199\u624d\u547d\u4e2d\u3001\u7c92\u5ea6\u53d8\u7ec6\u7684\u4ee3\u4ef7\u843d\u5728\u54ea\u91cc<\/li>\n<li>\u2713 \u62ff\u5230\u4e00\u6761\u6162 SQL&#xff0c;\u80fd\u6309\u516d\u79cd\u5178\u578b\u5931\u6548\u5199\u6cd5\u9010\u6761\u6392\u67e5\u5206\u533a\u88c1\u526a&#xff0c;\u5e76\u5199\u51fa\u4fee\u590d\u7248&#xff1b;\u80fd\u533a\u5206&#034;\u6ca1\u88c1&#034;\u4e0e&#034;\u88c1\u4e86\u4f46\u6ca1\u7528&#034;<\/li>\n<li>\u2713 \u8bf4\u6e05\u5217\u5f0f\u6587\u4ef6\u7684\u56db\u5c42\u7edf\u8ba1\u7c92\u5ea6&#xff08;\u6587\u4ef6\u5217\u7edf\u8ba1\u3001row-group \u7684 min\/max\u3001page \u7ea7\u7d22\u5f15&#xff09;\u5206\u522b\u88ab\u8c01\u4f7f\u7528&#xff0c;\u5e76\u77e5\u9053 Spark \u5b98\u65b9\u53ea\u7ed9\u4e86\u524d\u4e24\u5c42\u7684\u5f00\u5173&#xff0c;\u7b2c\u4e09\u5c42\u6ca1\u6709\u5b98\u65b9\u4f9d\u636e<\/li>\n<li>\u2713 \u80fd\u5224\u65ad\u4e00\u5f20\u8868\u8be5\u600e\u4e48\u6392&#xff1a;\u7ebf\u6027\u6392\u5e8f\u3001\u8868\u7ea7 WRITE ORDERED BY\u3001\u8fd8\u662f\u591a\u7ef4 Z-Order \/ Hilbert&#xff0c;\u4f9d\u636e\u662f\u67e5\u8be2\u5f62\u6001\u800c\u4e0d\u662f&#034;\u542c\u8bf4 Z-Order \u5feb&#034;<\/li>\n<li>\u2713 \u8bb2\u6e05 Paimon bucket \/ Iceberg bucket(n) \/ Hive bucket \u7684\u8bed\u4e49\u5dee\u5f02&#xff0c;\u4ee5\u53ca&#034;\u5206\u6876\u8868 join \u514d shuffle&#034;\u6210\u7acb\u7684\u524d\u63d0\u6761\u4ef6<\/li>\n<li>\u2713 \u5728\u540c\u4e00\u5f20\u8868\u7684\u56db\u79cd\u5e03\u5c40\u72b6\u6001\u4e0b\u91cf\u51fa\u626b\u63cf\u6587\u4ef6\u6570\u4e0e\u884c\u6570\u7684\u5dee&#xff0c;\u5e76\u4e14\u77e5\u9053\u4e3a\u4ec0\u4e48\u5728 20 \u4e07\u884c\u7684\u6570\u636e\u4e0a\u91cf\u4e0d\u51fa\u6765<\/li>\n<li>\u2713 \u9762\u5bf9\u4e00\u4e2a\u65b0\u9700\u6c42&#xff0c;\u80fd\u7528&#034;\u5206\u533a\u5b9a\u76ee\u5f55\u3001\u5206\u6876\u5b9a\u6587\u4ef6\u3001\u6392\u5e8f\u5b9a\u5757&#034;\u4e09\u53e5\u8bdd\u5199\u51fa DDL \u8349\u6848&#xff0c;\u5e76\u8bf4\u660e\u6bcf\u4e2a\u9009\u62e9\u7684\u4f9d\u636e<\/li>\n<\/ul>\n<hr \/>\n<h3>5.1 \u4e09\u79cd&#034;\u5206\u533a&#034;&#xff0c;\u4e3a\u4ec0\u4e48\u4f1a\u4e92\u76f8\u6df7\u6dc6<\/h3>\n<p>\u5148\u7ed9\u7ed3\u8bba&#xff0c;\u540e\u9762\u90fd\u5728\u89e3\u91ca\u8fd9\u53e5\u8bdd&#xff1a;<\/p>\n<p>**\u5b83\u4eec\u4e0d\u662f&#034;\u540c\u4e00\u4ef6\u4e8b\u7684\u4e09\u79cd\u5b9e\u73b0&#034;&#xff0c;\u800c\u662f\u4e09\u79cd\u4e0d\u540c\u7684\u7269\u7406\u627f\u8bfa\u3002**\u627f\u8bfa\u4e0d\u540c&#xff0c;\u6240\u4ee5\u8c13\u8bcd\u7684\u5199\u6cd5\u3001\u5931\u6548\u7684\u6837\u5b50\u3001\u53d8\u7ec6\u7684\u4ee3\u4ef7&#xff0c;\u5168\u90fd\u4e0d\u4e00\u6837\u3002<\/p>\n<p>\u8f66\u8054\u7f51\u8fd9\u6761\u94fe\u8def\u4e0a\u4e09\u79cd\u8868\u683c\u5f0f\u5e38\u5e74\u5171\u5b58&#xff1a;Flink \u5199\u7684 Paimon ODS\u3001Spark \u5199\u7684 Iceberg DWD\/DWS\u3001\u5b58\u91cf\u6570\u4ed3\u7559\u4e0b\u7684 Hive \u5206\u533a\u8868\u3002\u540c\u4e00\u6761&#034;\u6309\u5929\u67e5\u4e00\u53f0\u8f66&#034;\u7684 SQL \u6253\u5230\u4e09\u79cd\u8868\u4e0a&#xff0c;\u53ef\u80fd\u662f\u4e09\u79cd\u7ed3\u679c\u2014\u2014\u5feb\u3001\u6162\u3001\u6216\u8005\u9759\u6084\u6084\u5730\u8fd4\u56de\u7a7a\u3002\u7b2c\u4e09\u79cd\u6700\u5371\u9669&#xff0c;\u540e\u9762 5.2 \u6848\u4f8b 5 \u4e13\u95e8\u8bb2\u5b83\u3002<\/p>\n<h4>5.1.1 \u4e00\u5f20\u5bf9\u7167\u8868\u628a\u5dee\u5f02\u9489\u4f4f<\/h4>\n<p>\u8868 5-1 \u4e09\u79cd\u5206\u533a\u673a\u5236\u5bf9\u7167<\/p>\n<table>\n<tr>\u7ef4\u5ea6Hive \u98ce\u683c\u76ee\u5f55\u5206\u533aIceberg \u9690\u85cf\u5206\u533aPaimon \u5206\u533a &#043; Bucket<\/tr>\n<tbody>\n<tr>\n<td>\u5206\u533a\u5217\u662f\u5426\u771f\u5b9e\u5b58\u5728<\/td>\n<td>\u662f&#xff0c;\u662f\u8868\u7684\u4e00\u5217&#xff0c;SELECT \u80fd\u67e5\u5230<\/td>\n<td>\u5426&#xff0c;\u8868\u91cc\u53ea\u6709\u539f\u59cb\u5217&#xff08;report_date&#xff09;&#xff0c;\u5206\u533a\u5b57\u6bb5\u662f transform \u6d3e\u751f\u7684<\/td>\n<td>\u662f&#xff08;\u5982 __dt__&#xff09;&#xff0c;\u4e14\u901a\u5e38\u540c\u65f6\u662f\u4e3b\u952e\u7684\u4e00\u90e8\u5206<\/td>\n<\/tr>\n<tr>\n<td>\u7269\u7406\u5f62\u6001<\/td>\n<td>__dt__&#061;2026-10-15\/00001.parquet \u76ee\u5f55\u540d\u5373\u5206\u533a<\/td>\n<td>data\/report_date_day&#061;2026-10-15\/0000x.parquet&#xff0c;\u76ee\u5f55\u7531 transform \u751f\u6210&#xff0c;\u4f46\u7531\u8868\u683c\u5f0f\u7ba1\u7406&#xff0c;\u7528\u6237\u4e0d\u8be5\u6309\u76ee\u5f55\u540d\u5199\u6761\u4ef6<\/td>\n<td>__dt__&#061;2026-10-15\/bucket-0\/&#xff0c;\u4e24\u7ea7\u76ee\u5f55&#xff0c;bucket \u5b50\u76ee\u5f55\u53ef\u89c1<\/td>\n<\/tr>\n<tr>\n<td>\u8c01\u8d1f\u8d23\u88c1\u526a<\/td>\n<td>Spark \u81ea\u5df1&#xff1a;\u5411 metastore \u8981\u5206\u533a\u5217\u8868&#xff0c;\u7528\u8c13\u8bcd\u53bb\u5339\u914d\u5206\u533a\u503c<\/td>\n<td>\u8868\u683c\u5f0f planner&#xff1a;\u628a\u5bf9\u539f\u59cb\u5217\u7684\u8c13\u8bcd\u7ffb\u8bd1\u6210 transform \u57df\u4e0a\u7684\u7ea6\u675f&#xff0c;\u518d\u5728 manifest \u91cc\u6311\u5019\u9009\u6587\u4ef6<\/td>\n<td>Paimon scan&#xff1a;\u5206\u533a\u88c1\u526a &#043; \u5bf9 bucket-key \u7684\u7b49\u503c\u6761\u4ef6\u5b9a\u4f4d\u5230\u6876<\/td>\n<\/tr>\n<tr>\n<td>\u8c13\u8bcd\u600e\u4e48\u5199\u624d\u547d\u4e2d<\/td>\n<td>\u76f4\u63a5\u5bf9\u5206\u533a\u5217\u5199\u5e38\u91cf\u6bd4\u8f83&#xff1b;\u7c7b\u578b\u3001\u683c\u5f0f\u5fc5\u987b\u4e0e\u76ee\u5f55\u91cc\u7684\u503c\u4e00\u81f4<\/td>\n<td>\u5bf9\u539f\u59cb\u5217\u5199\u6bd4\u8f83\u5373\u53ef&#xff0c;\u65e0\u9700\u77e5\u9053\u5206\u533a\u5b57\u6bb5&#xff1b;\u4f46\u8868\u8fbe\u5f0f\u5fc5\u987b\u80fd\u843d\u56de\u8be5\u5217<\/td>\n<td>\u5206\u533a\u5217\u7b49\u503c\/\u8303\u56f4 \u2192 \u88c1\u5230\u5206\u533a&#xff1b;bucket-key \u7b49\u503c \u2192 \u624d\u80fd\u88c1\u5230\u6876<\/td>\n<\/tr>\n<tr>\n<td>\u5178\u578b\u5931\u6548<\/td>\n<td>\u51fd\u6570\u5305\u88f9\u5206\u533a\u5217\u3001\u9690\u5f0f\u7c7b\u578b\u8f6c\u6362\u3001\u5206\u533a\u503c\u683c\u5f0f\u4e0d\u4e00\u81f4<\/td>\n<td>\u8fc7\u6ee4\u5217\u4e0e\u5206\u533a transform \u7684\u8f93\u5165\u5217\u4e0d\u662f\u540c\u4e00\u5217&#xff1b;partition evolution \u540e\u65b0\u65e7 spec \u5e76\u5b58<\/td>\n<td>\u5bf9 bucket-key \u5199\u8303\u56f4\u6761\u4ef6&#xff08;&gt;\/BETWEEN&#xff09;\u53ea\u80fd\u5168\u6876\u626b<\/td>\n<\/tr>\n<tr>\n<td>\u53d8\u7ec6\u7684\u4ee3\u4ef7<\/td>\n<td>\u76ee\u5f55\u6570\u4e0e\u5206\u533a\u5217\u8868\u53d8\u957f&#xff0c;\u5217\u5206\u533a\u672c\u8eab\u6210\u4e3a\u74f6\u9888&#xff1b;\u5c0f\u6587\u4ef6\u968f\u5206\u533a\u6570\u7ebf\u6027\u589e\u52a0<\/td>\n<td>manifest \u6587\u4ef6\u4e0e\u5217\u8868\u81a8\u80c0&#xff0c;\u5143\u6570\u636e\u8bfb\u53d6\u65f6\u95f4\u4e0a\u5347<\/td>\n<td>\u5206\u533a\u6570 \u00d7 bucket \u6570 &#061; \u6587\u4ef6\u6570\u4e0b\u754c&#xff0c;\u6876\u6570\u6539\u8d77\u6765\u6700\u9ebb\u70e6&#xff08;\u56fa\u5b9a\u6876\u9700\u91cd\u5237&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u4e8b\u540e\u80fd\u4e0d\u80fd\u6539<\/td>\n<td>ADD\/DROP PARTITION \u6539\u5217\u8868&#xff1b;\u6539\u5206\u533a\u5217\u7b49\u4e8e\u91cd\u5efa\u8868<\/td>\n<td>Partition Evolution \u652f\u6301\u6362\u5206\u533a\u5b57\u6bb5&#xff0c;\u5386\u53f2\u6570\u636e\u4fdd\u7559\u65e7 spec<\/td>\n<td>\u5206\u533a\u5217\u4e0d\u53ef\u53d8&#xff1b;bucket \u6570\u53d8\u66f4\u9700\u8981 rescale\/\u91cd\u5237\u6570\u636e<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u4e09\u4e2a&#034;\u6240\u4ee5&#034;\u503c\u5f97\u5355\u72ec\u8bb0&#xff1a;<\/p>\n<p>\u2460 \u6240\u4ee5 Iceberg \u8868\u91cc\u67e5\u4e0d\u5230\u5206\u533a\u5217\u3002 \u7b2c 2 \u7ae0\u5efa\u7684 iceberg.demo.dws_vehicle_daily \u7528 PARTITIONED BY (months(report_date))&#xff0c;\u4f60 SELECT \u91cc\u5199\u4e0d\u51fa report_date_month\u2014\u2014\u90a3\u662f\u9690\u85cf\u5206\u533a\u5b57\u6bb5&#xff0c;\u5728\u5143\u6570\u636e\u91cc&#xff0c;\u4e0d\u5728 schema \u91cc\u3002\u60f3\u9a8c\u8bc1\u5206\u533a\u600e\u4e48\u5207\u7684&#xff0c;\u5f97\u53bb\u8bfb\u5143\u6570\u636e\u8868&#xff08;Spark \u4fa7\u7528\u70b9\u53f7\u8bbf\u95ee iceberg.demo.dws_vehicle_daily.partitions&#xff1b;\u524d\u4f5c\u786c\u7ea6\u675f&#xff1a;Flink \u4fa7\u5199 $partitions&#xff0c;\u4e24\u8005\u4e0d\u53ef\u6df7\u7528&#xff09;\u3002<\/p>\n<p>\u2461 \u6240\u4ee5 Paimon \u7684\u6876\u4e0d\u662f&#034;\u6027\u80fd\u65cb\u94ae&#034;&#xff0c;\u662f&#034;\u5b9a\u4f4d\u89c4\u5219&#034;\u3002 \u7b2c 2 \u7ae0\u90a3\u5f20 ODS \u8868 &#039;bucket&#039; &#061; &#039;4&#039;\u3001\u4e3b\u952e vin,report_date,__dt__\u3002\u67e5\u8be2\u91cc\u7ed9 vin \u7b49\u503c \u21d2 \u80fd\u5b9a\u5230\u4e00\u4e2a\u6876&#xff1b;\u7ed9 speed &gt; 60 \u21d2 \u4e0e\u6876\u65e0\u5173&#xff0c;\u56db\u4e2a\u6876\u5168\u5f00\u3002\u8fd9\u4e0d\u662f\u5b9e\u73b0\u7f3a\u9677&#xff0c;\u662f hash \u5206\u6876\u7684\u6570\u5b66\u6027\u8d28&#xff1a;hash \u53ea\u652f\u6301\u7b49\u503c\u3002<\/p>\n<p>\u2462 \u6240\u4ee5&#034;Hive \u5206\u533a\u6570\u522b\u592a\u591a&#034;\u8fd9\u6761\u8001\u7ecf\u9a8c&#xff0c;\u5728\u6e56\u8868\u4e0a\u6362\u4e86\u5f62\u5f0f\u4f46\u6ca1\u6d88\u5931\u3002 \u524d\u4f5c\u5728 Iceberg \u7cfb\u5217\u91cc\u5b9a\u4e0b\u7684\u53e3\u5f84\u662f&#xff1a;\u9690\u85cf\u5206\u533a\u9075\u5faa**\u201c\u51e0\u5343\u91cf\u7ea7\u5206\u533a\u201d**\u539f\u5219&#xff0c;\u5206\u533a\u7c92\u5ea6\u8fc7\u7ec6\u5bfc\u81f4\u5143\u6570\u636e\u81a8\u80c0\u3002\u76ee\u5f55\u5206\u533a\u81a8\u80c0\u7684\u662f metastore \u4e0e list \u8c03\u7528&#xff0c;\u9690\u85cf\u5206\u533a\u81a8\u80c0\u7684\u662f manifest&#xff1b;\u4e24\u8005\u90fd\u662f&#034;\u5143\u6570\u636e\u5148\u625b\u4e0d\u4f4f&#xff0c;\u67e5\u8be2\u624d\u6162&#034;\u3002\u6309\u5929\u5206\u533a\u8dd1\u4e09\u4e94\u5e74\u662f\u51e0\u5343\u91cf\u7ea7&#xff0c;\u5b89\u5168&#xff1b;\u6309\u5c0f\u65f6\u4e14\u591a\u7ef4&#xff08;\u5929 \u00d7 \u57ce\u5e02 \u00d7 \u8f66\u578b&#xff09;\u5f88\u5bb9\u6613\u76f4\u63a5\u4e0a\u4e07&#xff0c;\u90a3\u5c31\u8981\u91cd\u65b0\u60f3\u3002<\/p>\n<h4>5.1.2 \u4e09\u8005\u5404\u7ba1\u4e00\u6bb5\u7269\u7406\u5e03\u5c40<\/h4>\n<p>                       \u4e00\u5f20\u8868\u7684\u7269\u7406\u5e03\u5c40&#xff0c;\u4e09\u4ef6\u4e8b\u4f5c\u7528\u5728\u4e09\u4e2a\u5c42\u6b21<br \/>\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n \u2502                                                                  \u2502<br \/>\n \u2502  \u5206\u533a Partition        \u2192 \u51b3\u5b9a\u6570\u636e\u843d\u5230&#034;\u54ea\u4e2a\u76ee\u5f55 \/ \u54ea\u5957 spec&#034;        \u2502<br \/>\n \u2502    \u7c92\u5ea6&#xff1a;\u5929\u3001\u6708\u3001\u57ce\u5e02\u6bb5        \u547d\u4e2d\u4e00\u6761 \u21d2 \u6574\u4e2a\u76ee\u5f55\u6839\u672c\u4e0d\u88ab\u6253\u5f00      \u2502<br \/>\n \u2502      \u2502                                                           \u2502<br \/>\n \u2502      \u251c\u2500\u2500 \u5206\u6876 Bucket     \u2192 \u51b3\u5b9a\u5206\u533a\u5185\u7684&#034;\u6587\u4ef6\u5f52\u5c5e\u89c4\u5219&#034;              \u2502<br \/>\n \u2502      \u2502     hash(key) % N        \u547d\u4e2d \u21d2 \u53ea\u6253\u5f00\u7ea6 1\/N \u7684\u6587\u4ef6         \u2502<br \/>\n \u2502      \u2502     \u2502                   \u540c\u65f6\u7ed9 join \u63d0\u4f9b&#034;\u514d shuffle&#034;\u7684\u6027\u8d28  \u2502<br \/>\n \u2502      \u2502     \u2502                                                           \u2502<br \/>\n \u2502      \u2502     \u2514\u2500\u2500 \u6587\u4ef6 File   \u2192 \u5217\u5f0f\u5b58\u50a8\u7684\u6700\u5c0f\u7ba1\u7406\u5355\u4f4d                 \u2502<br \/>\n \u2502      \u2502            \u2502                                                \u2502<br \/>\n \u2502      \u2502            \u2514\u2500 \u6392\u5e8f Sort \/ Z-Order                            \u2502<br \/>\n \u2502      \u2502                  \u51b3\u5b9a&#034;\u4e00\u4e2a\u6587\u4ef6\u5185\u90e8\u884c\u7684\u5148\u540e\u987a\u5e8f&#034;              \u2502<br \/>\n \u2502      \u2502                  \u547d\u4e2d \u21d2 \u540c\u4e00\u6587\u4ef6\u5185\u8df3\u8fc7 row-group \/ page      \u2502<br \/>\n \u2502      \u2502                                                           \u2502<br \/>\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<\/p>\n<p>   \u88c1\u526a\u7c92\u5ea6\u7531\u5916\u5230\u5185&#xff1a;  \u76ee\u5f55  &gt;&gt;  \u6587\u4ef6  &gt;&gt;  \u5757&#xff08;row-group&#xff09;  &gt;&gt;  \u884c<br \/>\n   \u6bcf\u5f80\u91cc\u4e00\u5c42&#xff0c;\u5355\u4f4d\u5c0f\u4e00\u5230\u4e24\u4e2a\u6570\u91cf\u7ea7&#xff0c;\u4f46\u80fd\u88c1\u7684\u6761\u4ef6\u7c7b\u578b\u4e5f\u53d8\u5c11\u3002<\/p>\n<p>\u56fe 5-1 \u5206\u533a \u00d7 \u5206\u6876 \u00d7 \u6392\u5e8f\u5728\u7269\u7406\u5e03\u5c40\u4e0a\u7684\u4f5c\u7528\u4f4d\u7f6e<\/p>\n<p>\u8fd9\u5f20\u56fe\u540e\u9762\u53cd\u590d\u7528\u3002\u4e09\u6761\u8bfb\u6cd5&#xff1a;<\/p>\n<ul>\n<li>\u5916\u5c42\u88c1\u6389\u7684\u662f&#034;\u6839\u672c\u4e0d\u6253\u5f00&#034;&#xff0c;\u5185\u5c42\u88c1\u6389\u7684\u662f&#034;\u6253\u5f00\u4e86\u4f46\u8df3\u8fc7\u5757&#034;\u3002 \u524d\u8005\u7701\u7684\u662f list &#043; \u5143\u6570\u636e &#043; IO \u7684\u5168\u90e8&#xff0c;\u540e\u8005\u7701\u7684\u662f IO \u7684\u4e00\u90e8\u5206\u3002\u6240\u4ee5\u987a\u5e8f\u6c38\u8fdc\u662f&#xff1a;\u5148\u628a\u5206\u533a\u8bbe\u8ba1\u5bf9&#xff0c;\u518d\u8c08\u6392\u5e8f\u3002<\/li>\n<li>\u4e09\u5c42\u4e0d\u4e92\u76f8\u66ff\u4ee3\u3002 \u6709\u4eba\u60f3\u7528 Z-Order \u8865\u5206\u533a\u6ca1\u5206\u597d\u7684\u6d1e\u2014\u2014\u5b83\u80fd\u8865\u4e00\u70b9&#xff08;\u591a\u7ef4\u76f8\u5173\u6027&#xff09;&#xff0c;\u4f46\u8865\u4e0d\u4e86&#034;\u8fd9\u4e2a\u6587\u4ef6\u91cc 90% \u7684\u884c\u90fd\u4e0d\u8be5\u88ab\u8bfb&#034;\u8fd9\u79cd\u7ed3\u6784\u6027\u6d6a\u8d39\u3002\u53cd\u8fc7\u6765&#xff0c;\u5206\u533a\u5207\u5f97\u518d\u7ec6&#xff0c;\u6587\u4ef6\u5185\u90e8\u4e71\u5e8f&#xff0c;row-group \u7684 min\/max \u5c31\u5bbd\u5230\u6ca1\u6709\u4efb\u4f55\u88c1\u526a\u4ef7\u503c\u3002<\/li>\n<li>\u5206\u6876\u662f\u552f\u4e00\u540c\u65f6\u5f71\u54cd\u8bfb\u548c\u5199\u7684\u5c42\u3002 \u5b83\u65e2\u88c1\u6587\u4ef6&#xff0c;\u53c8\u51b3\u5b9a join\/aggregate \u9700\u4e0d\u9700\u8981 shuffle\u3002\u6240\u4ee5\u5b83\u7684\u9009\u62e9\u7a7a\u95f4\u6700\u5c0f&#xff1a;\u6876\u952e\u5fc5\u987b\u662f\u9ad8\u9891\u7b49\u503c\u5217&#xff0c;\u6876\u6570\u4e00\u65e6\u5b9a\u4e0b\u5c31\u5f88\u96be\u6539\u3002<\/li>\n<\/ul>\n<p>\u4e0e ck \u7b2c 4 \u7ae0\u7684\u5bf9\u5e94\u5173\u7cfb&#xff1a;ClickHouse \u7684&#034;\u5206\u533a\u952e \u2192 \u6392\u5e8f\u952e granule \u2192 \u8df3\u6570\u7d22\u5f15 \u2192 \u6295\u5f71&#034;&#xff0c;\u5728 Spark \u6e56\u8868\u4e0a\u5c31\u662f&#034;\u5206\u533a \u2192 \u6587\u4ef6\/row-group min-max \u2192 page index\/bloom \u2192 \u805a\u7c07\u6392\u5e8f&#034;\u3002\u5dee\u522b\u5728\u4e8e ClickHouse \u7684\u6392\u5e8f\u952e\u662f\u5199\u6b7b\u5728 part \u5185\u90e8\u7684\u5f3a\u7ea6\u675f&#xff08;granule \u8fb9\u754c\u9760 mark \u6587\u4ef6\u7cbe\u786e\u53ef\u67e5&#xff09;&#xff0c;\u800c\u6e56\u8868\u7684&#034;\u6392\u5e8f&#034;\u662f\u4e00\u6b21\u6027\u7684\u5199\u5165\u5e03\u5c40&#xff0c;\u5199\u5b8c\u5c31\u4e0d\u518d\u7ef4\u62a4\u987a\u5e8f\u2014\u2014\u5b83\u9760\u7684\u662f\u7edf\u8ba1\u4fe1\u606f&#xff0c;\u4e0d\u662f mark \u7d22\u5f15\u3002\u8fd9\u4e2a\u5dee\u522b\u5728 5.4.4 \u4f1a\u91cf\u5316\u3002<\/p>\n<hr \/>\n<h3>5.2 \u5206\u533a\u88c1\u526a&#xff1a;\u516d\u79cd\u5931\u6548\u5199\u6cd5<\/h3>\n<p>\u5148\u89e3\u51b3&#034;\u600e\u4e48\u77e5\u9053\u88c1\u6ca1\u88c1&#034;&#xff0c;\u518d\u8bb2\u516d\u79cd\u5931\u6548\u3002\u7b2c 4 \u7ae0 4.2.5 \u57cb\u7684\u90a3\u4e2a\u8d1f\u9762\u7ed3\u679c\u2014\u2014\u4e24\u6761\u67e5\u8be2\u8bfb\u4e86\u540c\u6837\u591a\u7684\u6587\u4ef6\u2014\u2014\u5c31\u662f\u4ece\u8fd9\u91cc\u5f00\u59cb\u67e5\u7684\u3002<\/p>\n<h4>5.2.1 \u4e09\u4e2a\u89c2\u6d4b\u70b9&#xff0c;\u6309\u53ef\u4fe1\u5ea6\u6392\u5e8f<\/h4>\n<table>\n<tr>\u89c2\u6d4b\u70b9\u600e\u4e48\u53d6\u56de\u7b54\u4ec0\u4e48\u5c40\u9650<\/tr>\n<tbody>\n<tr>\n<td>files read \/ scanned rows<\/td>\n<td>Spark UI \u7684 SQL \u9875&#xff0c;\u6216 SparkListener \u4e8b\u4ef6\u65e5\u5fd7&#xff08;\u7b2c 17 \u7ae0\u811a\u672c\u5316&#xff09;<\/td>\n<td>\u4e8b\u5b9e&#xff1a;\u5230\u5e95\u8bfb\u4e86\u591a\u5c11<\/td>\n<td>\u5fc5\u987b\u8dd1\u5b8c\u624d\u77e5\u9053&#xff1b;\u8dd1\u4e00\u6b21\u53ef\u80fd\u5c31\u662f\u5168\u8868\u626b<\/td>\n<\/tr>\n<tr>\n<td>PartitionFilters<\/td>\n<td>EXPLAIN FORMATTED \u91cc FileScan \u8282\u70b9\u7684\u8fd9\u4e00\u884c<\/td>\n<td>\u610f\u56fe&#xff1a;\u4f18\u5316\u5668\u6253\u7b97\u7528\u54ea\u4e9b\u6761\u4ef6\u88c1\u5206\u533a<\/td>\n<td>\u975e\u7a7a \u2260 \u547d\u4e2d\u3002\u5b57\u7b26\u4e32\u533a\u95f4\u5199\u9519\u683c\u5f0f\u65f6\u5b83\u7167\u6837\u975e\u7a7a<\/td>\n<\/tr>\n<tr>\n<td>SHOW PARTITIONS \/ \u5143\u6570\u636e\u8868<\/td>\n<td>SHOW PARTITIONS paimon.demo.ods_vehicle_telemetry&#xff1b;Iceberg \u67e5 $.partitions<\/td>\n<td>\u524d\u63d0&#xff1a;\u5206\u533a\u503c\u5b9e\u9645\u957f\u4ec0\u4e48\u6837<\/td>\n<td>\u53ea\u80fd\u4eba\u5de5\u6bd4\u5bf9&#xff0c;\u4e0d\u76f4\u63a5\u8bf4\u660e\u67e5\u8be2\u547d\u4e2d<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u4e09\u4e2a\u5fc5\u987b\u4e00\u8d77\u7528\u3002\u53ea\u770b PartitionFilters \u4f1a\u88ab\u6848\u4f8b 5 \u9a97&#xff1b;\u53ea\u8dd1\u4e00\u6b21\u770b UI \u53c8\u592a\u8d35\u3002\u751f\u4ea7\u4e0a\u7684\u6807\u51c6\u52a8\u4f5c\u662f&#xff1a;\u5148 SHOW PARTITIONS \u786e\u8ba4\u5206\u533a\u503c\u5f62\u6001&#xff0c;\u518d EXPLAIN \u786e\u8ba4\u8868\u8fbe\u5f0f\u5f62\u6001&#xff0c;\u6700\u540e\u7528 measure_scan.py&#xff08;5.6&#xff09;\u91cf\u4e00\u6b21\u771f\u5b9e\u6587\u4ef6\u6570\u3002<\/p>\n<p><span class=\"token comment\">&#8212; \u89c2\u6d4b\u70b9 3 \u7684\u56fa\u5b9a\u52a8\u4f5c&#xff1a;\u5148\u770b\u6e05\u5206\u533a\u503c\u5230\u5e95\u957f\u4ec0\u4e48\u6837<\/span><br \/>\n<span class=\"token keyword\">SHOW<\/span> PARTITIONS paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u8f93\u51fa\u5217&#xff1a;partition(STRING)&#xff0c;\u5f62\u5982 __dt__&#061;2026-10-15&#xff0c;\u5171 30 \u884c<\/span><br \/>\n<span class=\"token comment\">&#8212; \u9a8c\u6536&#xff1a;30 \u884c&#xff1b;\u503c\u5168\u90e8\u4e3a yyyy-MM-dd \u5f62\u5f0f&#xff1b;min&#061;2026-10-01&#xff0c;max&#061;2026-10-30<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff1a;SHOW PARTITIONS \u5bf9 Paimon\/Hive \u98ce\u683c\u7684\u663e\u5f0f\u5206\u533a\u5217\u76f4\u63a5\u5217\u76ee\u5f55&#xff1b;Iceberg \u7684\u9690\u85cf\u5206\u533a\u4e0d\u663e\u793a\u5728\u8fd9\u91cc&#xff0c;\u8981\u67e5\u5143\u6570\u636e\u8868\u3002<\/p>\n<p>\u8f93\u51fa\u8868\u7ed3\u6784&#xff1a;\u5355\u5217 partition(STRING)\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;\u884c\u6570\u7b49\u4e8e\u7b2c 2 \u7ae0\u8bb0\u5f55\u7684 30 \u4e2a\u5206\u533a&#xff1b;\u4efb\u4f55\u4e00\u884c\u7684\u503c\u5f62\u6001\u4e0e\u540e\u9762 SQL \u91cc\u5199\u7684\u5b57\u9762\u91cf\u4e0d\u4e00\u81f4&#xff0c;\u5c31\u5148\u522b\u5f80\u4e0b\u505a\u2014\u2014\u8fd9\u662f\u6848\u4f8b 5 \u7684\u5165\u53e3\u3002<\/p>\n<h4>5.2.2 \u8868 5-2 \u516d\u79cd\u5931\u6548\u901f\u67e5<\/h4>\n<p>\u8868 5-2 \u5206\u533a\u88c1\u526a\u5931\u6548\u901f\u67e5&#xff08;\u6bcf\u4e00\u884c\u540e\u9762\u90fd\u6709\u53ef\u8dd1\u7684\u5b9e\u9a8c&#xff09;<\/p>\n<table>\n<tr>#\u5931\u6548\u7c7b\u578b\u53cd\u4f8b\u7279\u5f81\u4e3a\u4ec0\u4e48\u4fee\u590d\u65b9\u5411\u89c2\u6d4b\u75c7\u72b6<\/tr>\n<tbody>\n<tr>\n<td>1<\/td>\n<td>\u51fd\u6570\u5305\u88f9\u5206\u533a\u5217<\/td>\n<td>to_date(__dt__) &#061; date&#039;&#8230;&#039;\u3001substr(__dt__,1,7)&#061;&#039;2026-10&#039;<\/td>\n<td>\u88c1\u526a\u62ff\u7684\u662f\u5206\u533a\u503c&#xff0c;\u4e0d\u662f\u8868\u8fbe\u5f0f\u7684\u503c&#xff1b;\u51fd\u6570\u6253\u65ad&#034;\u5217 \u2194 \u5e38\u91cf&#034;\u7684\u6bd4\u8f83<\/td>\n<td>\u628a\u51fd\u6570\u6362\u5230\u5e38\u91cf\u4fa7&#xff0c;\u6216\u76f4\u63a5\u5bf9\u5206\u533a\u5217\u5199\u533a\u95f4<\/td>\n<td>PartitionFilters \u4e3a\u7a7a&#xff0c;\u5168\u5206\u533a\u626b<\/td>\n<\/tr>\n<tr>\n<td>2<\/td>\n<td>\u7c7b\u578b\/\u5b57\u9762\u91cf\u4e0d\u5339\u914d<\/td>\n<td>\u5206\u533a\u5217 STRING \u5374\u5199 __dt__ &#061; 20261015&#xff08;\u6f0f\u5f15\u53f7&#xff09;<\/td>\n<td>\u9700\u8981\u4e00\u6b21\u9690\u5f0f\u8f6c\u6362&#xff0c;\u6bd4\u8f83\u5bf9\u8c61\u4e0d\u518d\u662f\u5206\u533a\u503c\u672c\u8eab<\/td>\n<td>\u6309\u5206\u533a\u5217\u771f\u5b9e\u7c7b\u578b\u5199\u5b57\u9762\u91cf<\/td>\n<td>\u53ef\u80fd&#034;\u80fd\u88c1\u4f46\u88c1\u9519&#034;&#xff0c;\u4e5f\u53ef\u80fd\u4e0d\u88c1<\/td>\n<\/tr>\n<tr>\n<td>3<\/td>\n<td>OR \u628a\u5206\u533a\u8c13\u8bcd\u964d\u7ea7<\/td>\n<td>__dt__&#061;&#039;2026-10-15&#039; OR city_id&#061;310007<\/td>\n<td>\u6790\u53d6\u5f0f\u7684\u53e6\u4e00\u652f\u4e0d\u542b\u5206\u533a\u5217&#xff0c;\u6574\u4f53\u65e0\u6cd5\u7ed9\u51fa\u5206\u533a\u8fb9\u754c<\/td>\n<td>\u62c6\u6210 UNION ALL&#xff0c;\u6216\u628a\u975e\u5206\u533a\u6761\u4ef6\u4e5f\u6362\u6210\u5206\u533a\u5217\u6761\u4ef6<\/td>\n<td>PartitionFilters \u4e3a\u7a7a<\/td>\n<\/tr>\n<tr>\n<td>4<\/td>\n<td>\u5426\u5b9a\u8c13\u8bcd&#xff08;&lt;&gt; \/ NOT IN \/ IS NOT NULL&#xff09;<\/td>\n<td>__dt__ &lt;&gt; &#039;2026-10-15&#039;<\/td>\n<td>\u901a\u5e38\u80fd\u88c1&#xff0c;\u4f46\u53ea\u80fd\u6392\u9664\u88ab\u70b9\u540d\u7684\u503c&#xff0c;\u5269\u4e0b\u5168\u8bfb<\/td>\n<td>\u6539\u6210\u663e\u5f0f\u6b63\u5411\u679a\u4e3e&#xff08;IN (\u2026)&#xff09;<\/td>\n<td>PartitionFilters \u975e\u7a7a\u4f46 files read \u51e0\u4e4e\u6ca1\u964d<\/td>\n<\/tr>\n<tr>\n<td>5<\/td>\n<td>\u5206\u533a\u503c\u683c\u5f0f\u4e0d\u4e00\u81f4<\/td>\n<td>\u76ee\u5f55\u662f 2026-10-15&#xff0c;\u6761\u4ef6\u5199 &#039;20261015&#039;<\/td>\n<td>\u88c1\u526a&#034;\u6210\u529f&#034;&#xff0c;\u547d\u4e2d 0 \u4e2a\u5206\u533a<\/td>\n<td>\u5199\u5165\u4fa7\u7edf\u4e00\u683c\u5f0f &#043; \u7ed3\u679c\u884c\u6570\u65ad\u8a00<\/td>\n<td>\u8fd4\u56de\u7a7a\u7ed3\u679c&#xff0c;\u4e0d\u62a5\u9519&#xff0c;\u901f\u5ea6\u8fd8\u5f88\u5feb<\/td>\n<\/tr>\n<tr>\n<td>6<\/td>\n<td>\u5206\u533a\u5217\u8fdb SELECT \u518d\u8fc7\u6ee4<\/td>\n<td>CTE\/\u5b50\u67e5\u8be2\u91cc\u7b97\u51fa dt_str \u522b\u540d&#xff0c;\u5916\u5c42 WHERE dt_str&#061;\u2026<\/td>\n<td>\u5916\u5c42\u8fc7\u6ee4\u7684\u662f\u6d3e\u751f\u5217&#xff1b;\u5c55\u5f00\u540e\u4ecd\u662f\u51fd\u6570\u5305\u88f9&#xff08;\u56de\u5230 #1&#xff09;<\/td>\n<td>\u628a\u8c13\u8bcd\u5199\u56de\u539f\u59cb\u5206\u533a\u5217&#xff0c;\u4e14\u5728 CTE \u5185\u5c42\u5c31\u8fc7\u6ee4<\/td>\n<td>PartitionFilters \u4e3a\u7a7a<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u7b2c 4 \u884c\u662f\u8fd9\u5f20\u8868\u91cc\u552f\u4e00&#034;\u53cd\u5e38\u8bc6&#034;\u7684&#xff1a;\u5b83\u4e0d\u662f\u5931\u6548&#xff0c;\u662f\u6536\u76ca\u5f52\u96f6\u3002 \u5f88\u591a\u4eba\u770b\u5230 PartitionFilters \u975e\u7a7a\u5c31\u4ee5\u4e3a\u4f18\u5316\u751f\u6548\u4e86&#xff0c;\u5b9e\u9645\u8bfb\u4e86 29\/30 \u4e2a\u5206\u533a\u3002\u8fd9\u4e00\u884c\u7684\u4ef7\u503c\u5728\u4e8e\u544a\u8bc9\u4f60&#xff1a;&#034;\u88c1\u526a\u751f\u6548&#034;\u8fd9\u4e2a\u7ed3\u8bba\u5fc5\u987b\u7531 files read \u7ed9\u51fa&#xff0c;\u4e0d\u80fd\u7531 EXPLAIN \u7ed9\u51fa\u2014\u2014\u8fd9\u5c31\u662f\u7b2c 4 \u7ae0 4.2 \u90a3\u6761&#034;Spark \u7684\u8ba1\u5212\u91cc\u53ea\u6709\u4f30\u7b97&#xff0c;\u6ca1\u6709\u5b9e\u6d4b&#034;\u7684\u5177\u4f53\u540e\u679c\u3002<\/p>\n<h4>5.2.3 \u516d\u4e2a\u6848\u4f8b\u9010\u4e2a\u8dd1<\/h4>\n<p>\u4ee5\u4e0b\u5168\u90e8\u57fa\u4e8e\u7b2c 2 \u7ae0\u7684 paimon.demo.ods_vehicle_telemetry&#xff08;20 \u4e07\u884c\u300130 \u4e2a\u5206\u533a&#xff09;\u3002\u6bcf\u4e2a\u6848\u4f8b\u7ed9\u300c\u53cd\u4f8b \u2192 \u4fee\u590d \u2192 \u5224\u636e\u300d\u3002\u5efa\u8bae\u628a\u516d\u6bb5\u4e00\u8d77\u5b58\u6210 pruning_fail_cases.sql&#xff0c;\u4f5c\u4e3a\u56e2\u961f\u7684\u8c13\u8bcd\u8bc4\u5ba1\u7528\u4f8b\u96c6\u3002<\/p>\n<p>\u6848\u4f8b 1&#xff1a;\u51fd\u6570\u5305\u88f9\u5206\u533a\u5217<\/p>\n<p><span class=\"token comment\">&#8212; \u53cd\u4f8b 1A&#xff1a;\u5bf9\u5206\u533a\u5217\u5957\u51fd\u6570<\/span><br \/>\n<span class=\"token keyword\">EXPLAIN<\/span> FORMATTED<br \/>\n<span class=\"token keyword\">SELECT<\/span> vin<span class=\"token punctuation\">,<\/span> <span class=\"token function\">avg<\/span><span class=\"token punctuation\">(<\/span>speed<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<br \/>\n<span class=\"token keyword\">WHERE<\/span> to_date<span class=\"token punctuation\">(<\/span>__dt__<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token keyword\">DATE<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span><br \/>\n<span class=\"token keyword\">GROUP<\/span> <span class=\"token keyword\">BY<\/span> vin<span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u53cd\u4f8b 1B&#xff1a;\u5bf9\u65f6\u95f4\u5217\u5957\u51fd\u6570&#xff08;\u8fd9\u5f20\u8868\u91cc report_date \u4e0d\u662f\u5206\u533a\u5217&#xff0c;__dt__ \u624d\u662f&#xff09;<\/span><br \/>\n<span class=\"token keyword\">WHERE<\/span> date_format<span class=\"token punctuation\">(<\/span>reported_at<span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;yyyy-MM-dd&#039;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u4fee\u590d 1A&#xff1a;\u628a\u51fd\u6570\u4ece\u5217\u4fa7\u632a\u8d70&#xff0c;\u76f4\u63a5\u6bd4\u8f83\u5206\u533a\u5217<\/span><br \/>\n<span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u4fee\u590d 1B&#xff1a;\u8d70\u771f\u6b63\u5b58\u5728\u7684\u5206\u533a\u5217&#xff0c;\u65f6\u95f4\u51fd\u6570\u53ea\u7528\u4e8e\u9009\u533a\u95f4<\/span><br \/>\n<span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">BETWEEN<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span> <span class=\"token operator\">AND<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span><br \/>\n  <span class=\"token operator\">AND<\/span> reported_at <span class=\"token operator\">&gt;&#061;<\/span> <span class=\"token keyword\">TIMESTAMP<\/span> <span class=\"token string\">&#039;2026-10-15 00:00:00&#039;<\/span><br \/>\n  <span class=\"token operator\">AND<\/span> reported_at <span class=\"token operator\">&lt;<\/span>  <span class=\"token keyword\">TIMESTAMP<\/span> <span class=\"token string\">&#039;2026-10-16 00:00:00&#039;<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff1a;1A \u7684\u5931\u6548\u539f\u56e0\u6700\u76f4\u767d\u2014\u2014\u88c1\u526a\u662f\u62ff\u5206\u533a\u503c&#xff08;\u5b57\u7b26\u4e32 2026-10-15&#xff09;\u53bb\u548c\u8c13\u8bcd\u6bd4\u8f83&#xff0c;\u5957\u4e0a to_date() \u540e\u4f18\u5316\u5668\u9700\u8981&#034;\u77e5\u9053&#034;\u51fd\u6570\u5728\u5206\u533a\u503c\u57df\u4e0a\u7684\u6620\u5c04&#xff0c;Spark \u4e0d\u505a\u8fd9\u79cd\u53cd\u6f14&#xff08;\u9664\u5c11\u6570\u5b98\u65b9\u5217\u51fa\u7684\u53ef\u4e0b\u63a8\u8868\u8fbe\u5f0f\u5916&#xff09;\u30021B \u662f\u751f\u4ea7\u91cc\u66f4\u9ad8\u9891\u7684\u7248\u672c&#xff1a;reported_at \u4e0d\u662f\u5206\u533a\u5217&#xff0c;__dt__ \u624d\u662f&#xff0c;\u8fc7\u6ee4\u5199\u9519\u5217 &#061; \u4e00\u5b9a\u5168\u8868\u626b&#xff0c;\u65e0\u8bba\u6709\u6ca1\u6709\u51fd\u6570\u3002<\/p>\n<p>\u4fee\u590d\u7248 1B \u7684\u5199\u6cd5\u6709\u4e2a\u523b\u610f\u7684\u8bbe\u8ba1&#xff1a;\u7528 &gt;&#061; \/ &lt; \u800c\u4e0d\u662f BETWEEN \u5305\u4f4f\u65f6\u523b\u5217&#xff0c;\u907f\u514d\u8fb9\u754c\u65e5\u7684\u91cd\u590d\u8ba1\u6570&#xff1b;__dt__ \u7684\u533a\u95f4\u6761\u4ef6\u4e0e\u5b83\u7b49\u4ef7&#xff0c;\u4fdd\u7559\u662f\u4e3a\u4e86\u8ba9&#034;\u5206\u533a\u5148\u88c1\u5230 1 \u4e2a&#034;\u8fd9\u4e2a\u610f\u56fe\u5728 SQL \u91cc\u663e\u5f0f\u53ef\u89c1\u3002<\/p>\n<p>\u8f93\u51fa\u8868\u7ed3\u6784&#xff1a;vin(STRING), avg(speed)(FLOAT)\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;\u53cd\u4f8b\u7684 FileScan \u8282\u70b9 PartitionFilters: \u8fd9\u4e00\u884c\u4e3a\u7a7a []&#xff0c;UI files read \u8986\u76d6 30 \u4e2a\u5206\u533a&#xff1b;\u4fee\u590d\u7248 PartitionFilters \u542b isnotnull(__dt__) \u4e0e\u7b49\u503c\/\u533a\u95f4\u6761\u4ef6&#xff0c;files read \u7ea6\u4e3a\u53cd\u4f8b\u7684 1\/30\u3002\u5982\u679c\u53cd\u4f8b\u5728\u4f60\u7684\u7248\u672c\u91cc PartitionFilters \u975e\u7a7a&#xff0c;\u8bf4\u660e\u8be5\u7248\u672c\u5df2\u80fd\u5bf9 to_date(\u5b57\u7b26\u4e32) \u505a\u5e38\u91cf\u6298\u53e0&#xff08;__dt__ \u662f\u5e38\u91cf\u65f6\u53ef\u6298&#xff09;&#xff0c;\u628a\u8fd9\u6761\u4f8b\u5916\u8bb0\u8fdb 5.2.6 \u7684\u5dee\u5f02\u8868&#xff0c;\u522b\u6539\u5224\u636e\u3002<\/p>\n<p>\u6848\u4f8b 2&#xff1a;\u7c7b\u578b\u4e0e\u5b57\u9762\u91cf\u4e0d\u5339\u914d<\/p>\n<p><span class=\"token comment\">&#8212; \u53cd\u4f8b&#xff1a;__dt__ \u662f STRING&#xff0c;\u7ed9\u4e86\u4e00\u4e2a\u6574\u6570<\/span><br \/>\n<span class=\"token keyword\">EXPLAIN<\/span> FORMATTED<br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<br \/>\n<span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">20261015<\/span><span class=\"token punctuation\">;<\/span>          <span class=\"token comment\">&#8212; \u6ce8\u610f&#xff1a;\u6ca1\u6709\u5f15\u53f7<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u4fee\u590d&#xff1a;\u6309\u5206\u533a\u5217\u7684\u771f\u5b9e\u7c7b\u578b\u5199<\/span><br \/>\n<span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u5bf9\u7167\u7ec4&#xff1a;\u540c\u4e00\u5f20 Iceberg \u8868&#xff08;\u5206\u533a\u5217\u662f DATE&#xff09;&#xff0c;\u5199\u6cd5\u53c8\u4e0d\u540c<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>dws_vehicle_daily<br \/>\n<span class=\"token keyword\">WHERE<\/span> report_date <span class=\"token operator\">&#061;<\/span> <span class=\"token keyword\">DATE<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span><span class=\"token punctuation\">;<\/span>    <span class=\"token comment\">&#8212; DATE \u5217\u914d DATE \u5b57\u9762\u91cf<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff1a;\u8fd9\u4e00\u7ec4\u7684\u91cd\u70b9\u4e0d\u662f&#034;\u4f1a\u4e0d\u4f1a\u62a5\u9519&#034;&#xff0c;\u800c\u662f\u540c\u4e00\u8bed\u4e49\u5728\u4e09\u79cd\u8868\u91cc\u5199\u6cd5\u4e0d\u540c&#xff1a;Hive \u98ce\u683c\u7684 __dt__ STRING \u7528\u5b57\u7b26\u4e32&#xff0c;Iceberg \u7684 months(report_date) \u7528\u539f\u59cb DATE \u5217&#xff0c;Paimon \u7684 __dt__ \u4e5f\u662f\u5b57\u7b26\u4e32\u4e14\u540c\u65f6\u662f\u4e3b\u952e\u7684\u4e00\u90e8\u5206\u3002\u8de8\u5f15\u64ce\u94fe\u8def\u91cc\u6700\u5e38\u89c1\u7684\u4e8b\u6545\u5c31\u662f&#034;Flink \u4fa7\u6309 yyyyMMdd \u5199\u3001Spark \u4fa7\u6309 yyyy-MM-dd \u8bfb&#034;\u2014\u2014\u5b83\u5c31\u662f\u6848\u4f8b 5\u3002<\/p>\n<p>ANSI \u76f8\u5173\u63d0\u9192&#xff1a;Spark 4.0 \u8d77 spark.sql.ansi.enabled \u9ed8\u8ba4 true&#xff08;\u7b2c 2 \u7ae0\u5df2\u53d6\u8bc1&#xff09;&#xff0c;\u5b57\u7b26\u4e32\u5411\u6570\u503c\/\u65e5\u671f\u7684\u9690\u5f0f\u8f6c\u6362\u5728 ANSI \u4e0b\u66f4\u5bb9\u6613\u76f4\u63a5\u629b\u9519\u800c\u4e0d\u662f\u9759\u9ed8\u7ed9 NULL\u3002\u8fd9\u662f\u597d\u4e8b\u2014\u2014\u4f46\u522b\u628a\u5b83\u5f53\u4fdd\u9669&#xff0c;\u7b2c 5 \u79cd\u5931\u6548&#xff08;\u683c\u5f0f\u4e0d\u4e00\u81f4&#xff09;\u5728 ANSI \u4e0b\u4f9d\u7136\u9759\u9ed8\u8fd4\u56de\u7a7a\u3002<\/p>\n<p>\u8f93\u51fa\u8868\u7ed3\u6784&#xff1a;count(BIGINT)\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;\u53cd\u4f8b\u7684\u8fd4\u56de\u503c\u8981\u4e48\u662f 0&#xff08;\u5f53\u6210\u4e0d\u547d\u4e2d&#xff09;\u3001\u8981\u4e48\u6574\u6bb5\u629b\u7c7b\u578b\u9519\u3002\u4e24\u79cd\u90fd\u53ef\u63a5\u53d7&#xff0c;\u4f46\u5fc5\u987b\u80fd\u89e3\u91ca\u662f\u54ea\u4e00\u79cd&#xff1b;\u4fee\u590d\u7248\u5e94\u8fd4\u56de\u8be5\u5206\u533a\u884c\u6570&#xff08;\u7ea6 6666 \u884c&#xff0c;20 \u4e07 \u00f7 30&#xff0c;\u4e3b\u952e\u53bb\u91cd\u540e\u6309\u5b9e\u9645\u8bb0\u5f55&#xff09;\u3002<\/p>\n<p>\u6848\u4f8b 3&#xff1a;OR \u628a\u5206\u533a\u8c13\u8bcd\u964d\u7ea7<\/p>\n<p><span class=\"token comment\">&#8212; \u53cd\u4f8b<\/span><br \/>\n<span class=\"token keyword\">EXPLAIN<\/span> FORMATTED<br \/>\n<span class=\"token keyword\">SELECT<\/span> vin<span class=\"token punctuation\">,<\/span> speed <span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<br \/>\n<span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span> <span class=\"token operator\">OR<\/span> city_id <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">310007<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u4fee\u590d A&#xff1a;\u62c6 UNION ALL&#xff0c;\u6bcf\u652f\u81ea\u5df1\u53ef\u88c1<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> vin<span class=\"token punctuation\">,<\/span> speed <span class=\"token keyword\">FROM<\/span> <span class=\"token punctuation\">.<\/span><span class=\"token punctuation\">.<\/span><span class=\"token punctuation\">.<\/span> <span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span><br \/>\n<span class=\"token keyword\">UNION<\/span> <span class=\"token keyword\">ALL<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> vin<span class=\"token punctuation\">,<\/span> speed <span class=\"token keyword\">FROM<\/span> <span class=\"token punctuation\">.<\/span><span class=\"token punctuation\">.<\/span><span class=\"token punctuation\">.<\/span> <span class=\"token keyword\">WHERE<\/span> city_id <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">310007<\/span> <span class=\"token operator\">AND<\/span> __dt__ <span class=\"token operator\">&lt;&gt;<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u4fee\u590d B&#xff1a;\u5982\u679c\u4e1a\u52a1\u771f\u5b9e\u610f\u56fe\u662f&#034;\u4e14&#034;&#xff0c;\u90a3\u5c31\u662f\u5199\u9519\u4e86&#xff0c;\u6539\u6210 AND<\/span><br \/>\n<span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span> <span class=\"token operator\">AND<\/span> city_id <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">310007<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff1a;\u6790\u53d6\u5f0f\u8981\u6c42\u6bcf\u4e00\u652f\u90fd\u80fd\u7ed9\u51fa\u5206\u533a\u8fb9\u754c&#xff0c;\u5426\u5219\u6574\u4f53\u6ca1\u6709\u4e0a\u754c\u3002\u4fee\u590d A \u91cc\u7b2c\u4e8c\u652f\u5e26\u4e86 __dt__ &lt;&gt; &#039;2026-10-15&#039;&#xff0c;\u662f\u4e3a\u4e86\u9632\u6b62\u4e24\u652f\u91cd\u53e0\u8fd4\u56de\u91cd\u590d\u884c&#xff08;UNION ALL \u4e0d\u53bb\u91cd&#xff09;\u2014\u2014\u8fd9\u4e2a\u5199\u6cd5\u628a\u4ee3\u4ef7\u8bf4\u5f97\u5f88\u6e05\u695a&#xff1a;OR \u7684\u53e6\u4e00\u652f\u5982\u679c\u8981\u6309\u975e\u5206\u533a\u5217\u67e5&#xff0c;\u5b83\u65e0\u8bba\u5982\u4f55\u662f\u5168\u5206\u533a\u626b&#xff0c;\u62c6\u4e0d\u62c6\u90fd\u4e00\u6837\u3002\u6240\u4ee5\u771f\u6b63\u7684\u51b3\u7b56\u662f\u4fee\u590d B&#xff1a;\u5148\u786e\u8ba4\u4e1a\u52a1\u662f\u4e0d\u662f\u8981 AND\u3002\u751f\u4ea7\u4e0a 90% \u7684\u8fd9\u7c7b OR \u662f\u7b14\u8bef\u3002<\/p>\n<p>\u8f93\u51fa\u8868\u7ed3\u6784&#xff1a;vin(STRING), speed(FLOAT)\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;\u53cd\u4f8b PartitionFilters \u4e3a\u7a7a\u4e14 files read \u2248 \u5168\u8868&#xff1b;\u4fee\u590d B \u7684 files read \u2248 1\/30\u3002\u4fee\u590d A \u53ea\u5e94\u5728\u7ed3\u679c\u786e\u5b9e\u8981&#034;\u5e76\u96c6&#034;\u65f6\u7528\u2014\u2014\u5b83\u628a\u4e00\u6b21\u5168\u8868\u626b\u6362\u6210\u4e86&#034;\u4e00\u6b21 1\/30 \u626b &#043; \u4e00\u6b21\u5168\u8868\u626b&#034;&#xff0c;\u6bd4\u53cd\u4f8b\u66f4\u6162&#xff0c;\u552f\u4e00\u6536\u76ca\u662f\u7b2c\u4e00\u652f\u7684\u7ed3\u679c\u80fd\u65e9\u51fa\u6765\u3002\u8fd9\u4e00\u6761\u5199\u5728\u8fd9\u91cc\u662f\u4e3a\u4e86\u8ba9\u8bfb\u8005\u660e\u767d&#xff1a;&#034;\u62c6 OR&#034;\u4e0d\u662f\u901a\u7528\u836f&#xff0c;\u5148\u95ee\u8bed\u4e49\u3002<\/p>\n<p>\u6848\u4f8b 4&#xff1a;\u5426\u5b9a\u8c13\u8bcd\u2014\u2014\u80fd\u88c1&#xff0c;\u4f46\u88c1\u4e0d\u52a8<\/p>\n<p><span class=\"token comment\">&#8212; &#034;\u6392\u9664\u4e00\u5929&#034;<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry <span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&lt;&gt;<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; &#034;\u6700\u8fd1 7 \u5929\u4e4b\u5916&#034;<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry <span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&gt;<\/span> <span class=\"token string\">&#039;2026-10-23&#039;<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u6b63\u5411\u679a\u4e3e\u4fee\u590d<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<br \/>\n<span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">IN<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;2026-10-24&#039;<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string\">&#039;2026-10-25&#039;<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string\">&#039;2026-10-26&#039;<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string\">&#039;2026-10-27&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                 <span class=\"token string\">&#039;2026-10-28&#039;<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string\">&#039;2026-10-29&#039;<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string\">&#039;2026-10-30&#039;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff1a;&lt;&gt; \u5728\u5b57\u7b26\u4e32\u5206\u533a\u4e0a\u80fd\u88c1&#xff08;\u6392\u9664\u88ab\u70b9\u540d\u7684\u5206\u533a\u503c&#xff09;&#xff0c;\u6240\u4ee5 PartitionFilters \u975e\u7a7a\u2014\u2014\u4f46\u53ea\u7701 1\/30\u3002\u53cd\u8fc7\u6765&#xff0c;&gt; \/ BETWEEN \/ IN (\u679a\u4e3e) \u8fd9\u4e9b\u80fd\u7ed9\u51fa\u6709\u9650\u96c6\u5408\u6216\u8fde\u7eed\u533a\u95f4\u7684\u8c13\u8bcd\u624d\u662f\u88c1\u526a\u559c\u6b22\u7684\u5f62\u72b6\u3002\u4fee\u590d\u7248\u548c &gt; \u7248\u7ed3\u679c\u76f8\u540c&#xff0c;\u533a\u522b\u5728\u4e8e\u679a\u4e3e\u8ba9&#034;\u8bfb 7 \u4e2a\u5206\u533a&#034;\u8fd9\u4ef6\u4e8b\u5728 SQL \u91cc\u53ef\u89c1&#xff0c;\u800c &gt; \u4f9d\u8d56\u4f18\u5316\u5668\u628a\u533a\u95f4\u7ffb\u8bd1\u6210\u5206\u533a\u96c6\u5408\u2014\u2014\u4e24\u79cd\u90fd\u80fd\u88c1&#xff0c;\u4f46\u679a\u4e3e\u5199\u6cd5\u5728\u8de8\u5f15\u64ce&#xff08;\u8d70 metastore \u5217\u5206\u533a&#xff09;\u65f6\u66f4\u7a33\u3002<\/p>\n<p>\u8f93\u51fa\u8868\u7ed3\u6784&#xff1a;count(BIGINT)\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;\u4e09\u4e2a\u67e5\u8be2\u5404\u81ea PartitionFilters \u90fd\u975e\u7a7a&#xff1b;\u5bf9\u6bd4 files read&#xff1a;&lt;&gt; \u7248 \u2248 29 \u4e2a\u5206\u533a&#xff0c;&gt; \u7248\u4e0e IN \u7248 \u2248 7 \u4e2a\u5206\u533a\u3002\u5982\u679c &lt;&gt; \u7248\u8bfb\u5230\u7684\u662f\u5168\u90e8\u5206\u533a\u800c\u4e0d\u662f 29 \u4e2a&#xff0c;\u8bf4\u660e\u8be5\u6e90\u7684\u5206\u533a\u5217\u8868\u83b7\u53d6\u672c\u8eab\u6ca1\u6709\u88ab\u8c13\u8bcd\u8fc7\u6ee4&#xff08;\u5e38\u89c1\u4e8e metastore \u7f13\u5b58\u6216\u67d0\u4e9b catalog \u5b9e\u73b0&#xff09;&#xff0c;\u628a\u8fd9\u6761\u8bb0\u8fdb 5.2.6\u3002<\/p>\n<p>\u6848\u4f8b 5&#xff1a;\u5206\u533a\u503c\u683c\u5f0f\u4e0d\u4e00\u81f4&#xff08;\u672c\u8868\u6700\u5371\u9669\u7684\u4e00\u4f8b&#xff09;<\/p>\n<p><span class=\"token comment\">&#8212; \u76ee\u5f55\u91cc\u662f 2026-10-15&#xff0c;\u6761\u4ef6\u5199\u6210\u7d27\u51d1\u683c\u5f0f<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry <span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;20261015&#039;<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u2191 \u8fd4\u56de 0 \u884c&#xff0c;\u4e0d\u62a5\u9519&#xff0c;\u6267\u884c\u5f97\u8fd8\u5f88\u5feb<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u9632\u5fa1\u6027\u5bf9\u8d26&#xff1a;\u5148\u770b\u5206\u533a\u503c\u5f62\u6001&#xff0c;\u518d\u5199\u67e5\u8be2<\/span><br \/>\n<span class=\"token keyword\">SHOW<\/span> PARTITIONS paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u4e0a\u7ebf\u65ad\u8a00&#xff08;Spark SQL \u91cc\u505a\u4e0b\u9650\u6821\u9a8c&#xff0c;\u8ba9&#034;0 \u884c&#034;\u53d8\u6210\u4e00\u6b21\u5931\u8d25&#xff09;<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token keyword\">if<\/span><span class=\"token punctuation\">(<\/span><span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span><br \/>\n          raise_error<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;\u5206\u533a 2026-10-15 \u65e0\u6570\u636e&#xff1a;\u68c0\u67e5 __dt__ \u683c\u5f0f\u4e0e\u5199\u5165\u4fa7\u662f\u5426\u4e00\u81f4&#039;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n          <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> rows_checked<br \/>\n<span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<br \/>\n<span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff1a;\u8fd9\u4e00\u4f8b\u548c\u524d\u56db\u4f8b\u6027\u8d28\u4e0d\u540c\u2014\u2014\u524d\u56db\u4f8b\u7684\u75c7\u72b6\u662f&#034;\u6162&#034;&#xff0c;\u8fd9\u4e00\u4f8b\u7684\u75c7\u72b6\u662f**\u201c\u5bf9\u4e0d\u4e0a\u8d26\u201d**\u3002\u88c1\u526a\u673a\u5236\u5b8c\u5168\u6b63\u5e38&#xff0c;\u5b83\u7cbe\u786e\u5730\u547d\u4e2d\u4e86 0 \u4e2a\u5206\u533a\u3002\u8f66\u8054\u7f51\u94fe\u8def\u91cc\u5b83\u7684\u53d1\u751f\u6761\u4ef6\u5f88\u5177\u4f53&#xff1a;Flink \u4fa7\u4f5c\u4e1a\u7528 yyyyMMdd \u5199 __DT__&#xff0c;Spark \u6279\u4fa7\u6309 yyyy-MM-dd \u8bfb&#xff1b;\u6216\u540c\u4e00\u6761\u94fe\u8def\u4e0a\u6e38\u6362\u4e86\u4e00\u6b21\u65f6\u95f4\u683c\u5f0f\u5316\u6a21\u5f0f\u3002\u7b2c 2 \u7ae0\u9020\u6570\u65f6\u523b\u610f\u8ba9 __dt__ \u4e0e report_date \u540c\u6e90\u3001\u5e76\u7edf\u4e00\u7528 date_format(&#8230;, &#039;yyyy-MM-dd&#039;)&#xff0c;\u5c31\u662f\u4e3a\u4e86\u628a\u8fd9\u7c7b\u95ee\u9898\u5728\u5b9e\u9a8c\u73af\u5883\u91cc\u6392\u9664\u6389&#xff0c;\u8ba9\u516d\u4e2a\u6848\u4f8b\u7684\u5176\u5b83\u53d8\u91cf\u5e72\u51c0\u3002<\/p>\n<p>raise_error \u662f Spark \u5185\u7f6e\u51fd\u6570&#xff0c;\u7528\u4e00\u884c SQL \u628a&#034;\u7a7a\u7ed3\u679c&#034;\u5347\u7ea7\u4e3a\u4f5c\u4e1a\u5931\u8d25&#xff0c;\u6bd4\u4e8b\u540e\u770b\u677f\u53d1\u73b0\u6570\u5b57\u4e3a 0 \u4fbf\u5b9c\u5f97\u591a\u3002\u751f\u4ea7\u4e0a\u8fd9\u7c7b\u65ad\u8a00\u5e94\u8be5\u51fa\u73b0\u5728\u6bcf\u4e2a\u6279\u4f5c\u4e1a\u7684\u5f00\u5934&#xff0c;\u800c\u4e0d\u662f\u6700\u540e\u3002<\/p>\n<p>\u8f93\u51fa\u8868\u7ed3\u6784&#xff1a;rows_checked(BIGINT)\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;\u7d27\u51d1\u683c\u5f0f\u90a3\u6761\u8fd4\u56de 0&#xff1b;\u65ad\u8a00\u7248\u5728\u6b63\u5e38\u5206\u533a\u4e0a\u8fd4\u56de\u7ea6 6666&#xff0c;\u628a\u65e5\u671f\u6539\u6210\u4e00\u4e2a\u4e0d\u5b58\u5728\u7684\u5206\u533a\u503c\u65f6\u5e94\u629b\u9519\u5e76\u7ec8\u6b62\u4f5c\u4e1a\u3002\u629b\u9519\u8fd9\u6761\u5fc5\u987b\u771f\u8dd1\u4e00\u6b21\u2014\u2014\u6ca1\u89c1\u8fc7\u5b83\u5931\u8d25\u7684\u56e2\u961f&#xff0c;\u7b49\u4e8e\u6ca1\u6709\u8fd9\u9053\u4fdd\u9669\u3002<\/p>\n<p>\u6848\u4f8b 6&#xff1a;\u5206\u533a\u5217\u8fdb\u4e86 SELECT \u518d\u8fc7\u6ee4<\/p>\n<p><span class=\"token comment\">&#8212; \u53cd\u4f8b&#xff1a;\u5728 CTE \u91cc\u9020\u6d3e\u751f\u5217&#xff0c;\u5916\u5c42\u8fc7\u6ee4\u6d3e\u751f\u5217<\/span><br \/>\n<span class=\"token keyword\">WITH<\/span> t <span class=\"token keyword\">AS<\/span> <span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token keyword\">SELECT<\/span> vin<span class=\"token punctuation\">,<\/span> speed<span class=\"token punctuation\">,<\/span> substr<span class=\"token punctuation\">(<\/span>__dt__<span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">7<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> month_str<br \/>\n    <span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<br \/>\n<span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> t <span class=\"token keyword\">WHERE<\/span> month_str <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;2026-10&#039;<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u4fee\u590d A&#xff1a;\u8c13\u8bcd\u5199\u56de\u539f\u59cb\u5206\u533a\u5217&#xff0c;\u7528\u533a\u95f4<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<br \/>\n<span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&gt;&#061;<\/span> <span class=\"token string\">&#039;2026-10-01&#039;<\/span> <span class=\"token operator\">AND<\/span> __dt__ <span class=\"token operator\">&lt;&#061;<\/span> <span class=\"token string\">&#039;2026-10-31&#039;<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u4fee\u590d B&#xff1a;CTE \u5185\u5c42\u5c31\u8fc7\u6ee4&#xff08;\u89c6\u56fe\/\u516c\u5171 CTE \u88ab\u590d\u7528\u65f6\u552f\u4e00\u53ef\u884c\u7684\u505a\u6cd5&#xff09;<\/span><br \/>\n<span class=\"token keyword\">WITH<\/span> t <span class=\"token keyword\">AS<\/span> <span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token keyword\">SELECT<\/span> vin<span class=\"token punctuation\">,<\/span> speed <span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<br \/>\n    <span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">&gt;&#061;<\/span> <span class=\"token string\">&#039;2026-10-01&#039;<\/span> <span class=\"token operator\">AND<\/span> __dt__ <span class=\"token operator\">&lt;&#061;<\/span> <span class=\"token string\">&#039;2026-10-31&#039;<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> t<span class=\"token punctuation\">;<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff1a;CTE \u4f1a\u88ab\u5185\u8054\u5c55\u5f00&#xff0c;\u6240\u4ee5\u5916\u5c42 month_str &#061; &#039;2026-10&#039; \u5c55\u5f00\u540e\u5c31\u662f substr(__dt__,1,7) &#061; &#039;2026-10&#039;\u2014\u2014\u56de\u5230\u4e86\u6848\u4f8b 1\u3002\u8fd9\u4e00\u4f8b\u771f\u6b63\u6559\u7684\u662f\u53e6\u4e00\u4ef6\u4e8b&#xff1a;\u8c13\u8bcd\u7684\u4f4d\u7f6e\u6bd4\u5199\u6cd5\u66f4\u5bb9\u6613\u51fa\u95ee\u9898\u3002\u89c6\u56fe\u3001\u516c\u5171 CTE\u3001\u88ab\u590d\u7528\u7684\u4e2d\u95f4\u5c42&#xff0c;\u53ea\u8981&#034;\u5206\u533a\u5217\u7684\u539f\u59cb\u5f62\u6001&#034;\u5728\u4f20\u9012\u8fc7\u7a0b\u4e2d\u88ab\u4e22\u6389&#xff08;\u53ea\u6295\u5f71\u4e86\u6d3e\u751f\u5217&#xff09;&#xff0c;\u4e0b\u6e38\u4f7f\u7528\u8005\u65e0\u8bba\u600e\u4e48\u5199\u90fd\u88c1\u4e0d\u52a8&#xff0c;\u800c\u4e14\u4ed6\u4e0d\u4f1a\u610f\u8bc6\u5230&#xff0c;\u56e0\u4e3a\u4ed6\u770b\u5230\u7684 schema \u91cc\u6839\u672c\u6ca1\u6709\u5206\u533a\u5217\u3002<\/p>\n<p>\u4fee\u590d B \u662f\u7ed9&#034;\u6211\u6539\u4e0d\u4e86\u89c6\u56fe\u5b9a\u4e49&#034;\u7684\u573a\u666f&#xff1a;\u4e0e\u5176\u5728\u4e0b\u6e38\u6323\u624e&#xff0c;\u4e0d\u5982\u7ed9\u89c6\u56fe\/CTE \u52a0\u4e00\u4e2a\u5fc5\u4f20\u53c2\u6570&#xff0c;\u6216\u8005\u628a\u4e2d\u95f4\u5c42\u843d\u6210\u5e26\u5206\u533a\u7684\u5c0f\u8868\u3002<\/p>\n<p>\u8f93\u51fa\u8868\u7ed3\u6784&#xff1a;count(BIGINT)\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;\u53cd\u4f8b\u4e0e\u4fee\u590d\u7248\u7ed3\u679c\u4e00\u81f4&#xff08;\u7ea6 20 \u4e07 \u00f7 30 \u00d7 31 \u5929\u53e3\u5f84\u5185\u5f53\u6708\u5168\u90e8\u6570\u636e&#xff0c;\u672c\u6570\u636e\u96c6\u4e3a 30 \u4e2a\u5206\u533a\u5168\u5728 10 \u6708&#xff0c;\u6545 \u2248 200000&#xff09;&#xff1b;files read \u4e0a\u53cd\u4f8b \u2248 30 \u4e2a\u5206\u533a&#xff0c;\u4fee\u590d\u7248\u540c\u6837 30 \u4e2a\u2014\u2014\u672c\u4f8b\u7684\u4fee\u590d\u7248\u4e0d\u4f1a\u5c11\u8bfb\u6587\u4ef6&#xff0c;\u56e0\u4e3a\u6570\u636e\u53ea\u6709 10 \u6708\u3002\u8fd9\u4e0d\u662f\u5931\u8d25&#xff1a;\u9a8c\u6536\u770b\u7684\u662f PartitionFilters \u662f\u5426\u975e\u7a7a&#xff08;\u673a\u5236\u5bf9\u4e0d\u5bf9&#xff09;&#xff0c;files read \u662f\u5426\u4e0b\u964d&#xff08;\u6536\u76ca\u6709\u6ca1\u6709&#xff09;\u3002\u8fd9\u4e24\u4e2a\u683c\u5b50\u5206\u5f00\u586b&#xff0c;\u6b63\u662f\u6848\u4f8b 4 \u6559\u7684\u90a3\u6761\u7eaa\u5f8b\u3002<\/p>\n<h4>5.2.4 \u4e00\u4e2a\u66f4\u9760\u4e0a\u6e38\u7684\u7ed3\u8bba&#xff1a;\u5206\u533a\u88c1\u526a\u7684\u5929\u82b1\u677f<\/h4>\n<p>\u516d\u4e2a\u6848\u4f8b\u90fd\u5728\u8bb2&#034;\u600e\u4e48\u5199\u8c13\u8bcd&#034;\u3002\u4f46\u771f\u6b63\u7684\u8bbe\u8ba1\u95ee\u9898\u662f&#xff1a;\u8fd9\u5f20\u8868\u7684\u5206\u533a\u5207\u5f97\u5bf9\u4e0d\u5bf9&#xff0c;\u8ba9\u88c1\u526a\u6709\u6ca1\u6709\u53ef\u80fd\u5b58\u5728\u3002<\/p>\n<p>   \u6309\u5929\u5206\u533a&#xff0c;20 \u4e07\u8f66 \u00d7 30 \u5929&#xff0c;\u67e5&#034;\u4e00\u53f0\u8f66\u4e00\u5929&#034;<br \/>\n       \u5206\u533a\u88c1\u526a \u21d2 \u547d\u4e2d 1\/30 \u76ee\u5f55           \u2190 \u6709\u6548<br \/>\n   \u6309\u5929\u5206\u533a&#xff0c;\u67e5&#034;\u4e00\u4e2a\u57ce\u5e02\u4e00\u5929&#034;<br \/>\n       \u5206\u533a\u88c1\u526a \u21d2 \u547d\u4e2d 1\/30&#xff1b;\u57ce\u5e02\u4ecd\u9760\u5217\u626b\u63cf  \u2190 \u4e00\u534a\u6709\u6548&#xff0c;\u53e6\u4e00\u534a\u8981\u770b 5.3<br \/>\n   \u6309\u5929\u5206\u533a&#xff0c;\u67e5&#034;\u4e00\u4e2a\u57ce\u5e02\u6700\u8fd1 30 \u5929&#034;<br \/>\n       \u5206\u533a\u88c1\u526a \u21d2 \u5168\u90e8\u547d\u4e2d&#xff0c;\u7b49\u4e8e\u6ca1\u88c1         \u2190 \u5931\u6548&#xff0c;\u4f46\u4e0d\u662f\u8c13\u8bcd\u5199\u9519<\/p>\n<p>\u7b2c\u4e09\u884c\u662f\u5173\u952e&#xff1a;\u67e5\u8be2\u6a21\u5f0f\u4e0e\u5206\u533a\u952e\u4e0d\u5339\u914d\u65f6&#xff0c;\u8c13\u8bcd\u5199\u5f97\u518d\u6b63\u786e\u4e5f\u88c1\u4e0d\u52a8\u3002 \u8fd9\u65f6\u8be5\u6539\u7684\u662f DDL&#xff08;\u52a0\u4e00\u5c42\u57ce\u5e02\u7ef4\u5ea6\u5206\u533a&#xff0c;\u6216\u8005\u7528 Z-Order \u628a\u57ce\u5e02\u4e0e\u65e5\u671f\u4e00\u8d77\u7f16\u8fdb\u76f8\u5173\u6027\u91cc&#xff0c;\u89c1 5.4&#xff09;&#xff0c;\u800c\u4e0d\u662f\u7ee7\u7eed\u6298\u817e SQL\u3002<\/p>\n<p>\u524d\u4f5c\u5728 ClickHouse \u4fa7\u5f97\u51fa\u8fc7\u540c\u6784\u7ed3\u8bba&#xff08;ck \u7b2c 4 \u7ae0 4.6&#xff1a;\u8df3\u8fc7\u6392\u5e8f\u952e\u7b2c\u4e00\u5217\u3001\u51fd\u6570\u5305\u5728\u952e\u5217\u4e0a&#xff0c;\u90fd\u662f&#034;\u8c13\u8bcd\u4e0e\u5e03\u5c40\u4e0d\u5339\u914d&#034;&#xff09;&#xff0c;Spark \u4fa7\u7684\u5dee\u522b\u53ea\u5728\u4e8e&#xff1a;\u5e03\u5c40\u6539\u4e86\u8981\u91cd\u5237\u6570\u636e&#xff0c;\u4ee3\u4ef7\u66f4\u5927&#xff0c;\u6240\u4ee5\u66f4\u9700\u8981\u4e8b\u524d\u5224\u65ad\u3002 \u5224\u65ad\u4f9d\u636e\u5c31\u662f 5.1.2 \u90a3\u5f20\u56fe\u7684\u4e09\u5c42&#xff1a;\u5148\u95ee\u8fd9\u4e2a\u67e5\u8be2\u80fd\u4e0d\u80fd\u843d\u5728\u67d0\u4e2a\u5206\u533a\u91cc&#xff0c;\u518d\u95ee\u80fd\u4e0d\u80fd\u843d\u5230\u67d0\u4e2a\u6876\u91cc&#xff0c;\u6700\u540e\u624d\u95ee\u6587\u4ef6\u5185\u6392\u5e8f\u80fd\u4e0d\u80fd\u5e2e\u5fd9\u3002<\/p>\n<h4>5.2.5 \u6e56\u8868\u7279\u6709\u7684\u7b2c\u4e03\u4ef6\u4e8b&#xff1a;\u5206\u533a\u88c1\u526a\u53d1\u751f\u5728&#034;\u54ea\u4e00\u5c42&#034;<\/h4>\n<p>Iceberg \u4e0e Paimon \u90fd\u81ea\u5e26\u5206\u533a\u8fc7\u6ee4\u80fd\u529b&#xff0c;Spark \u4fa7\u7684 PartitionFilters \u53ea\u662f\u628a\u6761\u4ef6\u4e0b\u63a8\u7ed9\u5b83\u4eec\u3002\u8fd9\u5e26\u6765\u4e00\u4e2a Hive \u8868\u6ca1\u6709\u7684\u884c\u4e3a&#xff1a;\u5019\u9009\u6587\u4ef6\u6311\u9009\u53d1\u751f\u5728\u8868\u683c\u5f0f\u7684 planner \u91cc&#xff0c;\u9700\u8981\u8bfb manifest \/ snapshot \u5143\u6570\u636e\u3002<\/p>\n<p>   Hive \u76ee\u5f55\u5206\u533a&#xff1a;  \u8c13\u8bcd \u2192 metastore \u5206\u533a\u5217\u8868\u5339\u914d \u2192 \u6253\u5f00\u76ee\u5f55<br \/>\n   Iceberg&#xff1a;        \u8c13\u8bcd \u2192 \u8bfb snapshot \u2192 \u8bfb manifest list \u2192 \u6309 partition \u7ea6\u675f<br \/>\n                    \u7b5b manifest \u2192 \u6309\u5217 min\/max \u518d\u7b5b\u6587\u4ef6 \u2192 \u624d\u8f6e\u5230 row-group<br \/>\n   Paimon&#xff1a;         \u8c13\u8bcd \u2192 \u8bfb snapshot \u2192 \u5206\u533a\u5217\u8868 \u2192 bucket \u5b9a\u4f4d \u2192 \u6587\u4ef6 \u2192 DataFile \u7edf\u8ba1<\/p>\n<p>\u6240\u4ee5\u540c\u4e00\u6761&#034;\u5206\u533a\u88c1\u526a\u6ca1\u751f\u6548&#034;\u7684\u7ed3\u8bba&#xff0c;\u5728\u4e09\u79cd\u8868\u4e0a\u7684\u6839\u56e0\u4f4d\u7f6e\u4e0d\u540c&#xff1a;Hive \u662f\u8c13\u8bcd\u8868\u8fbe\u95ee\u9898&#xff0c;Iceberg \u53ef\u80fd\u662f\u5143\u6570\u636e\u7f13\u5b58\u95ee\u9898&#xff08;\u524d\u4f5c\u786c\u7ea6\u675f&#xff1a;Iceberg \u5143\u6570\u636e\u7f13\u5b58\u9ed8\u8ba4 30 \u79d2&#xff0c;\u7ed3\u6784\u53d8\u66f4\u540e Flink \u4fa7\u9700\u91cd\u542f\u4f1a\u8bdd\u624d\u7acb\u5373\u751f\u6548\u2014\u2014Spark \u4fa7\u540c\u6837\u6709\u7f13\u5b58\u8bed\u4e49&#xff0c;\u8de8\u5f15\u64ce\u6539\u5b8c\u7acb\u523b\u67e5\u4e0d\u5230\u65b0\u6570\u636e\u8981\u60f3\u5230\u8fd9\u4e00\u5c42&#xff09;&#xff0c;Paimon \u53ef\u80fd\u662f bucket \u5b9a\u4f4d\u6ca1\u8d70\u7b49\u503c\u3002<\/p>\n<p>Hive \u8fd9\u4e00\u4fa7\u8fd8\u6709\u4e09\u4e2a\u5b98\u65b9\u5f00\u5173&#xff0c;\u89e3\u91ca\u4e86&#034;\u540c\u4e00\u53e5 SQL \u5728\u4e0d\u540c\u73af\u5883\u88c1\u5f97\u52a8\/\u88c1\u4e0d\u52a8&#034;\u7684\u4e00\u90e8\u5206\u5dee\u5f02&#xff08;\u9ed8\u8ba4\u503c\u9010\u5b57\u53d6\u81ea configuration \u9875&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u914d\u7f6e\u9ed8\u8ba4\u5b98\u65b9\u539f\u6587\u8981\u70b9<\/tr>\n<tbody>\n<tr>\n<td>spark.sql.hive.metastorePartitionPruning<\/td>\n<td>true&#xff08;1.5.0 \u8d77&#xff09;<\/td>\n<td>\u201csome predicates will be pushed down into the Hive metastore so that unmatching partitions can be eliminated earlier\u201d<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.hive.metastorePartitionPruningFastFallback<\/td>\n<td>false&#xff08;3.3.0 \u8d77&#xff09;<\/td>\n<td>\u5f00\u542f\u540e\u6539\u4e3a&#034;\u5148\u53d6\u56de\u5206\u533a\u540d&#xff0c;\u518d\u5728\u5ba2\u6237\u7aef\u7b97\u8fc7\u6ee4\u8868\u8fbe\u5f0f&#034;&#xff1b;\u542b TimeZoneAwareExpression \u7684\u8c13\u8bcd\u4e0d\u652f\u6301\u8fd9\u6761\u8def<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.hive.manageFilesourcePartitions<\/td>\n<td>\u89c1\u914d\u7f6e\u9875<\/td>\n<td>\u5b98\u65b9\u539f\u6587\u628a\u4e24\u8005\u8fde\u8d77\u6765&#xff1a;\u201cuse the metastore to prune partitions during query planning when spark.sql.hive.metastorePartitionPruning is set to true\u201d<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u8bfb\u8fd9\u4e09\u884c\u7684\u6b63\u786e\u65b9\u5f0f&#xff1a;&#034;\u628a\u8c13\u8bcd\u4e0b\u63a8\u8fdb metastore \u8ba9\u5b83\u65e9\u6dd8\u6c70&#034;\u548c&#034;\u5148\u628a\u5206\u533a\u540d\u5168\u635e\u56de\u6765\u518d\u81ea\u5df1\u7b5b&#034;\u662f\u4e24\u6761\u8def\u5f84&#xff0c;\u524d\u8005\u7701\u7684\u662f\u7f51\u7edc\u4e0e\u5206\u533a\u6570&#xff0c;\u540e\u8005\u7701\u7684\u662f\u517c\u5bb9\u6027\u95ee\u9898&#xff08;\u590d\u6742\u8c13\u8bcd\u3001MetaException \u9000\u8def&#xff09;\u3002\u5f53\u4f60\u7684 Hive \u5206\u533a\u8868\u7a81\u7136\u53d8\u6162\u3001\u6216\u6362\u4e86 metastore \u7248\u672c\u540e\u88c1\u526a\u884c\u4e3a\u53d8\u4e86&#xff0c;FastFallback \u662f\u7b2c\u4e00\u4e2a\u8be5\u770b\u7684\u5730\u65b9\u3002\u53e6\u5916\u8981\u7834\u9664\u4e00\u4e2a\u540d\u5b57&#xff1a;\u793e\u533a\u6587\u7ae0\u91cc\u6d41\u4f20\u7684 spark.sql.hive.metastore.partition.prune.enabled \u4e0d\u662f\u5b98\u65b9\u914d\u7f6e\u540d&#xff0c;\u5b98\u65b9\u540d\u662f\u4e0a\u9762\u8fd9\u4e2a\u3002<\/p>\n<p>\u8fd9\u6761\u5dee\u5f02\u7684\u5de5\u7a0b\u4ef7\u503c&#xff1a;\u9047\u5230&#034;\u6211\u660e\u660e\u88c1\u4e86\u4f46\u6ca1\u53d8\u5feb&#034;&#xff0c;\u5148\u95ee\u8fd9\u5f20\u8868\u7684\u88c1\u526a\u53d1\u751f\u5728\u54ea\u4e00\u5c42&#xff0c;\u518d\u51b3\u5b9a\u662f\u6539 SQL \u8fd8\u662f\u67e5\u5143\u6570\u636e\u3002<\/p>\n<h4>5.2.6 \u628a\u89c2\u5bdf\u7ed3\u679c\u843d\u6210\u7248\u672c\u5dee\u5f02\u8868<\/h4>\n<p>\u516d\u4e2a\u6848\u4f8b\u8dd1\u5b8c&#xff0c;\u628a\u4e0e\u9884\u671f\u4e0d\u7b26\u7684\u6761\u76ee\u8bb0\u4e0b\u6765\u3002\u8fd9\u5f20\u8868\u662f\u672c\u7cfb\u5217\u7559\u7ed9\u8bfb\u8005\u7684\u586b\u7a7a&#xff0c;\u4e5f\u662f\u56e2\u961f wiki \u4e0a\u6700\u6709\u7528\u7684\u4e00\u9875&#xff1a;<\/p>\n<table>\n<tr>\u6848\u4f8b\u9884\u671f\u5b9e\u6d4b&#xff08;\u586b\u81ea\u5df1\u7684\u7248\u672c\u4e0e\u7ed3\u679c&#xff09;\u5904\u7f6e<\/tr>\n<tbody>\n<tr>\n<td>1A to_date(__dt__)<\/td>\n<td>\u4e0d\u88c1<\/td>\n<td><\/td>\n<td>\u82e5\u88c1\u4e86&#xff1a;\u6ce8\u660e Spark \u7248\u672c &#043; catalog \u7248\u672c&#xff0c;\u8bc4\u5ba1\u65f6\u4e0d\u518d\u4f9d\u8d56&#034;\u5fc5\u7136\u5931\u6548&#034;<\/td>\n<\/tr>\n<tr>\n<td>2 STRING \u5206\u533a &#061; \u6574\u6570<\/td>\n<td>\u62a5\u9519\u6216 0 \u884c<\/td>\n<td><\/td>\n<td>\u7edf\u4e00\u6539\u4e3a\u663e\u5f0f\u5b57\u7b26\u4e32\u5b57\u9762\u91cf&#xff0c;\u65e0\u8bba\u54ea\u79cd\u75c7\u72b6<\/td>\n<\/tr>\n<tr>\n<td>3 OR \u6df7\u6742<\/td>\n<td>\u4e0d\u88c1<\/td>\n<td><\/td>\n<td>\u82e5\u90e8\u5206\u88c1&#xff1a;\u4ecd\u6309&#034;\u4e0d\u53ef\u4f9d\u8d56&#034;\u5904\u7406&#xff0c;hint \u4e0d\u7b97\u8bbe\u8ba1<\/td>\n<\/tr>\n<tr>\n<td>4 &lt;&gt;<\/td>\n<td>\u88c1 1\/30<\/td>\n<td><\/td>\n<td>\u82e5\u5168\u8bfb&#xff1a;\u8be5\u6e90\u7684\u5206\u533a\u5217\u8868\u83b7\u53d6\u672a\u88ab\u8c13\u8bcd\u8fc7\u6ee4&#xff0c;\u6539\u7528\u6b63\u5411\u679a\u4e3e<\/td>\n<\/tr>\n<tr>\n<td>5 \u683c\u5f0f\u4e0d\u4e00\u81f4<\/td>\n<td>0 \u884c\u4e0d\u62a5\u9519<\/td>\n<td><\/td>\n<td>\u4e00\u5f8b\u52a0 raise_error \u65ad\u8a00&#xff0c;\u4e0e\u5b9e\u6d4b\u65e0\u5173<\/td>\n<\/tr>\n<tr>\n<td>6 CTE \u6d3e\u751f\u5217<\/td>\n<td>\u4e0d\u88c1<\/td>\n<td><\/td>\n<td>\u4e00\u5f8b\u8981\u6c42\u4e2d\u95f4\u5c42\u66b4\u9732\u539f\u59cb\u5206\u533a\u5217<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>5.3 \u6587\u4ef6\u7ea7\u4e0e\u5217\u7ea7\u88c1\u526a&#xff1a;\u5206\u533a\u4e4b\u540e\u7684\u4e09\u5c42<\/h3>\n<p>\u5206\u533a\u88c1\u6389\u7684\u662f&#034;\u6574\u4e2a\u76ee\u5f55&#034;\u3002\u76ee\u5f55\u91cc\u5269\u4e0b\u7684&#xff0c;\u5c31\u8981\u9760\u5217\u5f0f\u6587\u4ef6\u81ea\u5df1\u7684\u7edf\u8ba1\u4fe1\u606f\u6765\u88c1\u3002\u8fd9\u4e00\u5c42\u7684\u80fd\u529b\u5168\u90e8\u5efa\u7acb\u5728&#034;\u6570\u636e\u662f\u6309\u67d0\u79cd\u987a\u5e8f\u5199\u7684&#034;\u8fd9\u4e2a\u524d\u63d0\u4e0a\u2014\u2014\u6240\u4ee5\u5b83\u548c 5.4 \u7684\u6392\u5e8f\u662f\u4e00\u5bf9&#xff1a;5.3 \u8bb2\u5c3a\u5b50&#xff0c;5.4 \u8bb2\u600e\u4e48\u628a\u6570\u636e\u6446\u5230\u5c3a\u5b50\u4e0a\u3002<\/p>\n<h4>5.3.1 \u4e00\u4e2a Parquet \u6587\u4ef6\u91cc\u6709\u56db\u5c42\u53ef\u4ee5\u88ab\u88c1\u7684\u4e1c\u897f<\/h4>\n<p>   demo.parquet<br \/>\n   \u251c\u2500\u2500 row-group #1            \u2190 Iceberg \u9ed8\u8ba4 write.parquet.row-group-size-bytes &#061; 128MB<br \/>\n   \u2502     \u251c\u2500\u2500 column chunk (vin)         \u5e26 statistics&#xff1a;min \/ max \/ null_count \/ value_count<br \/>\n   \u2502     \u251c\u2500\u2500 column chunk (report_date) \u5e26 statistics<br \/>\n   \u2502     \u2514\u2500\u2500 column chunk (speed)<br \/>\n   \u2502           \u2514\u2500\u2500 page \u2026                \u2190 \u9ed8\u8ba4 write.parquet.page-size-bytes &#061; 1MB<br \/>\n   \u2502                 \u2514\u2500\u2500 page \u7ea7\u7edf\u8ba1&#xff08;column index \/ page index&#xff09;<br \/>\n   \u251c\u2500\u2500 row-group #2<br \/>\n   \u2514\u2500\u2500 footer                  \u2190 schema &#043; \u6bcf\u4e2a row-group \u6bcf\u5217\u7684\u7edf\u8ba1\u5165\u53e3<\/p>\n<p>\u8868 5-3 \u56db\u5c42\u7edf\u8ba1\u3001\u4f7f\u7528\u65b9\u4e0e\u5f00\u5173&#xff08;\u9ed8\u8ba4\u503c\u9010\u5b57\u53d6\u81ea\u5b98\u65b9\u9875\u9762&#xff09;<\/p>\n<table>\n<tr>\u7c92\u5ea6\u5b58\u4ec0\u4e48\u8c01\u80fd\u7528\u5b83\u88c1\u76f8\u5173\u914d\u7f6e&#xff08;\u9ed8\u8ba4\u503c \/ \u8d77\u59cb\u7248\u672c&#xff09;<\/tr>\n<tbody>\n<tr>\n<td>\u5206\u533a&#xff08;\u76ee\u5f55\/spec&#xff09;<\/td>\n<td>\u5206\u533a\u503c<\/td>\n<td>\u8868\u683c\u5f0f planner\u3001metastore<\/td>\n<td>\u89c1 5.1\u30015.2<\/td>\n<\/tr>\n<tr>\n<td>\u6587\u4ef6<\/td>\n<td>\u6bcf\u5217\u7684 lower_bounds \/ upper_bounds \/ value_counts \/ null_value_counts \/ nan_value_counts \/ column_sizes<\/td>\n<td>Iceberg planner&#xff1a;\u5148\u6309\u5206\u533a\u88c1&#xff0c;\u518d\u6309\u5217\u7edf\u8ba1\u88c1\u6587\u4ef6<\/td>\n<td>write.metadata.metrics.default &#061; truncate(16)<\/td>\n<\/tr>\n<tr>\n<td>row-group \/ column chunk<\/td>\n<td>\u8be5\u5757\u5185\u8be5\u5217\u7684 min\/max\/null count<\/td>\n<td>Parquet reader&#xff08;\u8c13\u8bcd\u4e0b\u63a8\u540e\u9010\u5757\u5224\u5b9a\u53ef\u5426\u8df3\u8fc7&#xff09;<\/td>\n<td>spark.sql.parquet.filterPushdown &#061; true&#xff08;1.2.0 \u8d77&#xff09;&#xff1b;spark.sql.orc.filterPushdown &#061; true<\/td>\n<\/tr>\n<tr>\n<td>page<\/td>\n<td>page \u7ea7 min\/max \u4e0e\u4f4d\u7f6e\u7d22\u5f15<\/td>\n<td>Spark \u4fa7\u672a\u627e\u5230\u4efb\u4f55\u5b98\u65b9\u5f00\u5173&#xff08;\u89c1\u4e0b&#xff09;<\/td>\n<td>\u65e0\u5bf9\u5e94\u914d\u7f6e\u9879<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u7b2c\u4e09\u884c\u5230\u7b2c\u56db\u884c\u4e4b\u95f4\u6709\u4e2a\u5fc5\u987b\u5199\u6e05\u695a\u7684\u4e8b\u5b9e&#xff1a;<\/p>\n<p>\u672c\u6b21\u5728 Spark \u5b98\u65b9\u6587\u6863&#xff08;sql-data-sources-parquet.html\u3001configuration.html\u3001sql-performance-tuning.html \u4e09\u9875\u5168\u6587&#xff09;\u6ca1\u6709\u627e\u5230\u4efb\u4f55 page index \/ column index \/ page-level \u7684\u5b57\u6837&#xff0c;\u4e5f\u6ca1\u6709 spark.sql.parquet.columnIndexPageSize \u8fd9\u7c7b\u914d\u7f6e\u9879\u3002 Parquet \u683c\u5f0f\u672c\u8eab\u5b9a\u4e49\u4e86\u9875\u7ea7\u7d22\u5f15&#xff0c;Iceberg \u5199\u4fa7\u4e5f\u80fd\u751f\u6210&#xff0c;\u4f46**&#034;Spark \u8bfb\u53d6\u65f6\u4f1a\u4e0d\u4f1a\u7528\u5b83\u88c1 page&#034;\u8fd9\u4ef6\u4e8b\u6ca1\u6709\u5b98\u65b9\u4f9d\u636e**&#xff0c;\u672c\u7cfb\u5217\u4e0d\u628a\u5b83\u5199\u6210\u4e8b\u5b9e\u3002\u9700\u8981\u5b83\u65f6&#xff0c;\u7528 5.3.5 \u7684\u5b9e\u6d4b\u6cd5\u91cf\u7ed9\u81ea\u5df1\u770b\u3002<\/p>\n<p>\u540c\u6837\u503c\u5f97\u8bb0\u5f55\u7684\u4e00\u6761\u53d6\u8bc1\u7ed3\u8bba&#xff1a;\u793e\u533a\u6587\u7ae0\u91cc\u5e38\u5199\u7684 spark.sql.parquet.recordColumnFilter \u4e0d\u662f\u5b98\u65b9\u914d\u7f6e\u540d&#xff0c;\u5b98\u65b9\u5b58\u5728\u7684\u662f spark.sql.parquet.recordLevelFilter.enabled&#xff08;\u9ed8\u8ba4 false&#xff0c;2.3.0 \u8d77&#xff09;\u3002\u5b83\u7684\u4e00\u53e5\u8bdd\u63cf\u8ff0\u6b63\u597d\u66b4\u9732\u4e86\u4e00\u4e2a\u771f\u5b9e\u53d6\u820d&#xff1a;<\/p>\n<p>\u201cIf true, enables Parquet\u2019s native record-level filtering using the pushed down filters. This configuration only has an effect when spark.sql.parquet.filterPushdown is enabled and the vectorized reader is not used.\u201d<\/p>\n<h4>5.3.2 \u5411\u91cf\u5316\u8bfb\u53d6\u4e0e record-level filter \u4e92\u65a5&#xff0c;\u8fd9\u4e0d\u662f\u7ec6\u8282<\/h4>\n<p>\u4e0a\u9762\u90a3\u53e5\u5b98\u65b9\u539f\u6587\u91cc\u6709&#034;and the vectorized reader is not used&#034;\u3002\u628a\u5b83\u548c spark.sql.parquet.enableVectorizedReader &#061; true&#xff08;2.0.0 \u8d77&#xff0c;\u201cEnables vectorized parquet decoding.\u201d&#xff09;\u653e\u5728\u4e00\u8d77\u8bfb&#xff0c;\u7ed3\u8bba\u662f&#xff1a;<\/p>\n<p>   \u9ed8\u8ba4\u72b6\u6001&#xff1a;\u5411\u91cf\u5316 reader \u5f00 \u2192 record-level filtering \u5f00\u4e86\u4e5f\u4e0d\u751f\u6548&#xff08;\u5b98\u65b9\u660e\u8bf4&#xff09;<br \/>\n   \u5173\u6389\u5411\u91cf\u5316&#xff1a;\u80fd\u542f\u7528 record-level filtering&#xff0c;\u4f46\u9000\u5230\u975e\u5411\u91cf\u5316\u89e3\u7801\u8def\u5f84<br \/>\n   \u21d2 \u8fd9\u4e0d\u662f&#034;\u591a\u5f00\u4e00\u4e2a\u5f00\u5173\u591a\u4e00\u5c42\u88c1\u526a&#034;&#xff0c;\u662f\u4e00\u6b21\u8def\u5f84\u66ff\u6362<\/p>\n<p>\u600e\u4e48\u786e\u8ba4\u81ea\u5df1\u8d70\u7684\u662f\u54ea\u6761\u8def&#xff1a;EXPLAIN FORMATTED \u91cc FileScan parquet \u8282\u70b9\u6709\u4e00\u884c Batched: true\/false&#xff08;\u7b2c 4 \u7ae0 4.2 \u7684\u56db\u4e2a\u5fc5\u770b\u6807\u8bb0\u4e4b\u4e00&#xff09;\u3002Batched: false \u5c31\u662f\u6389\u5230\u4e86\u975e\u5411\u91cf\u5316\u8def\u5f84\u2014\u2014\u5e38\u89c1\u8bf1\u56e0\u662f schema \u91cc\u6709\u5411\u91cf\u5316 reader \u4e0d\u652f\u6301\u7684\u7c7b\u578b\u7ec4\u5408&#xff08;\u5b8c\u6574\u6e05\u5355\u672c\u6b21\u672a\u9010\u5b57\u53d6\u8bc1&#xff0c;\u6309 4.1 Parquet \u9875\u6838\u5bf9&#xff1b;\u8f66\u8054\u7f51\u8868\u91cc\u6700\u5bb9\u6613\u89e6\u53d1\u7684\u662f DECIMAL \u9ad8\u7cbe\u5ea6\u5217\u4e0e TIMESTAMP_NTZ \u6df7\u7528&#xff09;\u3002<\/p>\n<p>\u5224\u636e&#xff1a;\u540c\u4e00\u8c13\u8bcd&#xff0c;\u5206\u522b\u5728 enableVectorizedReader&#061;true&#xff08;\u9ed8\u8ba4&#xff09;\u4e0e false \u4e24\u79cd\u8bbe\u7f6e\u4e0b\u8dd1&#xff0c;\u6bd4\u8f83 UI \u7684 scanned rows \u4e0e bytes read\u3002\u53ea\u6709\u5f53 scanned rows \u660e\u663e\u4e0b\u964d\u65f6\u624d\u503c\u5f97\u6362\u8def\u5f84&#xff0c;\u5426\u5219\u5c31\u662f\u62ff\u89e3\u7801\u901f\u5ea6\u6362\u88c1\u526a\u7cbe\u5ea6\u3002<\/p>\n<h4>5.3.3 \u5b57\u7b26\u4e32\u5217\u7684 min\/max \u53ea\u8bb0\u524d 16 \u4e2a\u5b57\u7b26\u2014\u2014\u8fd9\u5bf9 VIN \u610f\u5473\u7740\u4ec0\u4e48<\/h4>\n<p>Iceberg \u5b98\u65b9 write.metadata.metrics.default &#061; truncate(16)&#xff08;\u53ef\u9009 none \/ counts \/ truncate(length) \/ full&#xff09;\u3002\u8fd9\u6761\u9ed8\u8ba4\u503c\u76f4\u63a5\u51b3\u5b9a\u5b57\u7b26\u4e32\u5217\u80fd\u4e0d\u80fd\u88c1\u6587\u4ef6&#xff1a;<\/p>\n<p>   \u672c\u6570\u636e\u96c6\u7684 VIN \u5f62\u6001&#xff1a;LSGKA12P7MA &#043; 6 \u4f4d\u6570\u5b57&#xff08;\u7b2c 2 \u7ae0\u9020\u6570&#xff09;<br \/>\n   \u524d 11 \u4f4d &#061; \u8f66\u578b\u56fa\u5b9a\u524d\u7f00&#xff0c;\u7b2c 12 \u4f4d\u8d77\u624d\u6709\u533a\u5206\u5ea6<br \/>\n   truncate(16) \u21d2 \u53c2\u4e0e\u7edf\u8ba1\u7684\u662f&#034;\u524d\u7f00 &#043; \u524d 5 \u4f4d\u6570\u5b57&#034;  \u2192 \u533a\u5206\u5ea6\u8db3\u591f&#xff0c;\u80fd\u88c1<\/p>\n<p>   \u4f46\u5982\u679c\u6362\u6210&#xff1a;<br \/>\n     \u2022 \u524d\u7f00\u5f88\u957f\u7684\u7f16\u7801&#xff08;\u5982\u7edf\u4e00 17 \u4f4d\u91cc\u53ea\u6709\u540e 2 \u4f4d\u53d8\u5316&#xff09;  \u2192 min\/max \u533a\u95f4\u51e0\u4e4e\u8986\u76d6\u5168\u90e8\u503c \u21d2 \u88c1\u4e0d\u52a8<br \/>\n     \u2022 \u6216\u8005\u5bf9 URL \/ \u62a5\u6587\u6587\u672c\u8fd9\u7c7b\u957f\u5b57\u7b26\u4e32\u505a\u524d\u7f00\u8fc7\u6ee4      \u2192 \u540c\u7406\u5931\u6548<br \/>\n   \u21d2 \u9ad8\u4ef7\u503c\u5217\u663e\u5f0f\u8bbe full&#xff0c;\u4ee3\u4ef7\u662f\u5143\u6570\u636e\u53d8\u5927<\/p>\n<p>\u8fd9\u6761\u7684\u5b9e\u7528\u4ef7\u503c\u5728\u4e8e\u5b83\u80fd\u63d0\u524d\u89e3\u91ca\u4e00\u79cd&#034;\u6392\u5e8f\u505a\u4e86\u4f46\u6ca1\u53d8\u5feb&#034;\u7684\u73b0\u8c61&#xff1a;\u4f60\u628a city_id&#xff08;INT&#xff09;\u6392\u597d\u4e86&#xff0c;\u88c1\u526a\u751f\u6548&#xff1b;\u4f46\u67e5\u8be2\u8d70\u7684\u662f vin LIKE &#039;LSGKA%&#039; \u8fd9\u79cd\u524d\u7f00\u6761\u4ef6&#xff0c;\u800c\u7edf\u8ba1\u88ab\u622a\u65ad\u2014\u2014\u6392\u5e8f\u5f62\u6210\u7684\u987a\u5e8f\u6027\u548c\u7edf\u8ba1\u80fd\u8868\u8fbe\u7684\u8fb9\u754c\u662f\u4e24\u56de\u4e8b\u3002<\/p>\n<h4>5.3.4 \u5c0f\u6587\u4ef6\u6162\u5728\u54ea&#xff1a;\u4e09\u4e2a\u4e92\u4e0d\u76f8\u5e72\u7684\u4ee3\u4ef7<\/h4>\n<p>\u5f88\u591a\u4eba\u628a&#034;\u5c0f\u6587\u4ef6\u6162&#034;\u5f52\u56e0\u6210&#034;\u6587\u4ef6\u591a&#034;&#xff0c;\u8fd9\u53ea\u5bf9\u4e09\u5206\u4e4b\u4e00\u3002\u4e09\u4e2a\u4ee3\u4ef7\u5404\u81ea\u6709\u72ec\u7acb\u7684\u5904\u7f6e\u65b9\u6cd5&#xff1a;<\/p>\n<p>\u2460 \u6253\u5f00\u6210\u672c&#xff08;\u6bcf\u6587\u4ef6\u4e00\u6b21 IO \u7a81\u53d1&#xff0c;\u4e0e\u6587\u4ef6\u5927\u5c0f\u65e0\u5173&#xff09;<br \/>\n   \u5b98\u65b9\u5bf9\u5206\u7247\u7684\u4f30\u7b97\u662f\u628a&#034;\u6253\u5f00\u6210\u672c&#034;\u6298\u6210\u5b57\u8282\u53c2\u4e0e\u88c5\u7bb1&#xff1a;<br \/>\n   spark.sql.files.openCostInBytes &#061; 4194304&#xff08;4 MB&#xff0c;2.0.0 \u8d77&#xff09;<br \/>\n   &#034;The estimated cost to open a file, measured by the number of bytes<br \/>\n    that could be scanned in the same time\u2026&#034;<br \/>\n   spark.sql.files.maxPartitionBytes &#061; 134217728&#xff08;128 MB&#xff09;<br \/>\n   \u21d2 \u4e00\u4e2a 1KB \u7684\u6587\u4ef6\u5728\u88c5\u7bb1\u65f6\u88ab\u5f53\u4f5c\u7ea6 4MB \u8ba1 \u21d2 \u6bcf\u4e2a task \u4ecd\u8981\u6253\u5f00\u51e0\u5341\u4e2a\u6587\u4ef6&#xff0c;<br \/>\n     \u6bcf\u4e2a\u6587\u4ef6\u4e00\u6b21 footer \u8bfb\u53d6&#xff08;footer \u5728\u6587\u4ef6\u672b\u5c3e&#xff0c;\u53e6\u4e00\u6b21\u968f\u673a IO&#xff09;<\/p>\n<p>\u2461 \u8ba1\u5212\u6210\u672c&#xff08;\u8868\u683c\u5f0f\u7684 planning \u65f6\u95f4\u968f\u6587\u4ef6\u6570\u7ebf\u6027\u589e\u957f&#xff09;<br \/>\n   Iceberg&#xff1a;manifest \u91cc\u6bcf\u4e2a\u6570\u636e\u6587\u4ef6\u4e00\u884c\u8bb0\u5f55 &#043; \u82e5\u5e72\u5217\u7edf\u8ba1&#xff08;\u8868 5-3 \u7b2c\u4e8c\u884c\u90a3\u4efd\u6e05\u5355&#xff09;<br \/>\n   \u21d2 10 \u4e07\u6587\u4ef6 &#061; 10 \u4e07\u6761 entry&#xff0c;planning \u4e0e\u5143\u6570\u636e\u7f13\u5b58\u90fd\u88ab\u63a8\u5927<br \/>\n   \u21d2 \u8fd9\u6b63\u662f\u524d\u4f5c&#034;\u51e0\u5343\u91cf\u7ea7\u5206\u533a&#034;\u7ecf\u9a8c\u5728\u6587\u4ef6\u7ef4\u5ea6\u7684\u53e6\u4e00\u534a&#xff1a;\u5206\u533a\u6570\u7ba1\u76ee\u5f55&#xff0c;\u6587\u4ef6\u6570\u7ba1 manifest<\/p>\n<p>\u2462 \u89e3\u7801\u6210\u672c&#xff08;\u5411\u91cf\u5316 reader \u6309\u6279\u89e3\u7801&#xff09;<br \/>\n   \u5411\u91cf\u5316\u8def\u5f84\u7684\u6536\u76ca\u6765\u81ea&#034;\u4e00\u6b21\u5904\u7406\u4e00\u6279\u884c&#034;&#xff1b;\u5c0f\u6587\u4ef6\u5f80\u5f80\u51d1\u4e0d\u6ee1\u4e00\u6279&#xff0c;<br \/>\n   \u6279\u7684\u8fb9\u754c\u4e0e\u6587\u4ef6\u8fb9\u754c\u53cd\u590d\u5bf9\u9f50&#xff0c;\u9000\u5316\u660e\u663e<br \/>\n   \u21d2 \u5177\u4f53\u6279\u5927\u5c0f\u914d\u7f6e\u540d\u672c\u6b21\u672a\u9010\u5b57\u53d6\u8bc1&#xff0c;\u5224\u636e\u770b\u5b9e\u6d4b&#xff1a;bytes read \u76f8\u8fd1\u4f46\u8017\u65f6\u5dee\u4e00\u4e2a\u91cf\u7ea7&#xff0c;<br \/>\n     \u57fa\u672c\u5c31\u662f\u8fd9\u4e00\u9879<\/p>\n<p>\u6cbb\u7406\u5165\u53e3\u53ea\u6709\u4e00\u4e2a&#xff0c;\u53c2\u6570\u6709\u4e09\u4e2a&#xff08;Iceberg 1.12 \u5b98\u65b9\u9ed8\u8ba4\u503c&#xff0c;\u9010\u5b57&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u53c2\u6570\u5b98\u65b9\u9ed8\u8ba4\u4f5c\u7528<\/tr>\n<tbody>\n<tr>\n<td>target-file-size-bytes<\/td>\n<td>536870912&#xff08;512 MB&#xff09;<\/td>\n<td>\u5408\u5e76\u76ee\u6807\u5927\u5c0f<\/td>\n<\/tr>\n<tr>\n<td>min-file-size-bytes<\/td>\n<td>75% of target file size<\/td>\n<td>\u5c0f\u4e8e\u6b64\u503c\u624d\u88ab\u89c6\u4e3a&#034;\u8be5\u88ab\u91cd\u5199&#034;<\/td>\n<\/tr>\n<tr>\n<td>max-file-size-bytes<\/td>\n<td>180% of target file size<\/td>\n<td>\u5927\u4e8e\u6b64\u503c\u4e5f\u53ef\u80fd\u88ab\u91cd\u5199&#xff08;\u62c6\u5206\/\u5408\u5e76&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>min-input-files<\/td>\n<td>5<\/td>\n<td>\u4e00\u7ec4\u81f3\u5c11\u51e0\u4e2a\u6587\u4ef6\u624d\u542f\u52a8\u91cd\u5199<\/td>\n<\/tr>\n<tr>\n<td>rewrite-all<\/td>\n<td>false<\/td>\n<td>\u9ed8\u8ba4\u4e0d\u91cd\u5199\u5927\u6587\u4ef6<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u8fd9\u5f20\u8868\u89e3\u91ca\u4e86\u751f\u4ea7\u4e0a\u6700\u5e38\u89c1\u7684\u56f0\u60d1&#xff1a;\u201c\u6211\u8dd1\u4e86 rewrite_data_files&#xff0c;\u6587\u4ef6\u6570\u6ca1\u53d8\u201d\u3002\u9ed8\u8ba4\u53c2\u6570\u4e0b\u5b83\u53ea\u5904\u7406 \u5c0f\u4e8e 384MB&#xff08;512\u00d775%&#xff09;\u4e14\u540c\u7ec4\u4e0d\u5c11\u4e8e 5 \u4e2a \u7684\u6587\u4ef6\u2014\u201410 \u4e07\u4e2a\u5c0f\u6587\u4ef6\u5f53\u7136\u4f1a\u88ab\u5408\u5e76&#xff0c;\u4f46 3 \u4e2a 600MB \u7684\u6587\u4ef6\u4e0d\u4f1a\u88ab\u52a8\u3002\u8981\u6539\u884c\u4e3a\u5c31\u663e\u5f0f\u4f20\u53c2&#xff0c;\u4e0d\u8981\u6307\u671b\u9ed8\u8ba4\u503c\u3002 \u7b2c 10 \u7ae0\u4f1a\u7ed9\u5b8c\u6574\u7684\u7ef4\u62a4\u4f5c\u4e1a\u7f16\u6392\u3002<\/p>\n<h4>5.3.5 \u89c2\u6d4b\u5217\u7ea7\u88c1\u526a&#xff1a;\u4e09\u4ef6\u5957\u52a0\u4e00\u6b21\u5bf9\u7167\u5b9e\u9a8c<\/h4>\n<p><span class=\"token comment\">&#8212; \u4e3b\u5b9e\u9a8c&#xff1a;\u540c\u4e00\u5f20\u8868&#xff0c;\u4e24\u4e2a\u8c13\u8bcd\u5206\u522b\u6253\u5728&#034;\u53c2\u4e0e\u6392\u5e8f\u7684\u5217&#034;\u4e0e&#034;\u4e0d\u53c2\u4e0e\u4efb\u4f55\u5e03\u5c40\u7684\u5217&#034;\u4e0a<\/span><br \/>\n<span class=\"token comment\">&#8212; \u524d\u7f6e&#xff1a;\u672c\u5b9e\u9a8c\u7528\u7684\u8868\u5728 5.4.3 \u5efa&#xff0c;\u5148\u8dd1 5.4 \u518d\u56de\u6765\u505a\u8fd9\u4e00\u9879\u4e5f\u53ef\u4ee5&#xff08;\u987a\u5e8f\u4e0d\u5f71\u54cd\u7ed3\u8bba&#xff09;<\/span><br \/>\n<span class=\"token keyword\">EXPLAIN<\/span> FORMATTED<br \/>\n<span class=\"token keyword\">SELECT<\/span> vin<span class=\"token punctuation\">,<\/span> speed <span class=\"token keyword\">FROM<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>dwd_vehicle_telemetry<br \/>\n<span class=\"token keyword\">WHERE<\/span> city_id <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">310007<\/span><span class=\"token punctuation\">;<\/span>               <span class=\"token comment\">&#8212; (a) \u53c2\u4e0e\u6392\u5e8f\/\u7edf\u8ba1\u7684\u5217<\/span><\/p>\n<p><span class=\"token keyword\">EXPLAIN<\/span> FORMATTED<br \/>\n<span class=\"token keyword\">SELECT<\/span> vin<span class=\"token punctuation\">,<\/span> speed <span class=\"token keyword\">FROM<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>dwd_vehicle_telemetry<br \/>\n<span class=\"token keyword\">WHERE<\/span> battery_pct <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">42<\/span><span class=\"token punctuation\">;<\/span>               <span class=\"token comment\">&#8212; (b) \u4e0d\u53c2\u4e0e\u6392\u5e8f\u7684\u5217&#xff08;\u57fa\u6570\u4f4e\u3001\u6563\u5e03\u5168\u8868&#xff09;<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff1a;\u4e24\u4e2a\u8c13\u8bcd\u90fd\u662f\u7b49\u503c\u3001\u90fd\u53ea\u5f15\u7528\u5355\u5217&#xff0c;\u56e0\u6b64\u90fd\u80fd\u88ab\u4e0b\u63a8&#xff08;PushedFilters \u90fd\u4f1a\u975e\u7a7a&#xff09;\u3002\u5dee\u522b\u53ea\u5728\u7edf\u8ba1\u8fb9\u754c\u80fd\u4e0d\u80fd\u7ed9\u51fa&#034;\u8fd9\u4e00\u5757\u4e00\u5b9a\u4e0d\u5339\u914d&#034;\u7684\u7ed3\u8bba\u3002(a) \u82e5\u6570\u636e\u6309 city_id \u6392\u8fc7&#xff0c;\u5757\u7ea7 min\/max \u5f88\u7a84&#xff1b;(b) battery_pct \u5728\u6bcf\u4e2a\u5757\u91cc\u90fd\u53d6\u904d 10~99&#xff0c;min\/max \u6c38\u8fdc\u8986\u76d6 42&#xff0c;\u4e00\u4e2a\u5757\u90fd\u8df3\u4e0d\u6389\u3002\u8fd9\u6b63\u662f&#034;\u8c13\u8bcd\u4e0b\u63a8\u751f\u6548 \u2260 \u6570\u636e\u88ab\u88c1\u6389&#034;\u7684\u6210\u56e0&#xff0c;\u4e5f\u662f\u7b2c 4 \u7ae0\u4e94\u95ee\u6a21\u677f\u7b2c 2 \u95ee\u8981\u770b\u5f97\u6bd4 PushedFilters \u66f4\u7ec6\u7684\u539f\u56e0\u3002<\/p>\n<p>\u8f93\u51fa\u8868\u7ed3\u6784&#xff1a;vin(STRING), speed(FLOAT)\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;\u4e24\u6761 SQL \u7684 PushedFilters \u90fd\u975e\u7a7a&#xff1b;\u5dee\u522b\u5fc5\u987b\u4f53\u73b0\u5728\u8fd0\u884c\u540e\u7684\u6570\u5b57\u4e0a\u2014\u2014(a) \u7684 scanned rows \u5e94\u663e\u8457\u4f4e\u4e8e (b)\u3002\u5982\u679c (a) \u4e5f\u6ca1\u964d&#xff0c;\u56de\u5230 5.4 \u68c0\u67e5\u6392\u5e8f\u662f\u5426\u771f\u7684\u843d\u5230\u4e86\u6587\u4ef6\u4e0a&#xff08;\u5386\u53f2\u6570\u636e\u4e0d\u4f1a\u56e0\u4e3a\u6539\u4e86\u4e00\u6761 ALTER &#8230; WRITE ORDERED BY \u800c\u91cd\u6392&#xff09;\u3002<\/p>\n<p>\u53e6\u5916\u4e24\u4e2a\u5b98\u65b9&#034;\u80fd\u51cf\u5c11\u626b\u63cf&#034;\u7684\u5f00\u5173&#xff0c;\u52a1\u5fc5\u548c\u6587\u4ef6\u7ea7\u88c1\u526a\u533a\u5206\u5f00&#xff1a;<\/p>\n<table>\n<tr>\u5f00\u5173&#xff08;\u9010\u5b57\u9ed8\u8ba4\u503c&#xff09;\u5b98\u65b9\u63cf\u8ff0\u8981\u70b9\u51cf\u7684\u662f\u4ec0\u4e48<\/tr>\n<tbody>\n<tr>\n<td>spark.sql.optimizer.dynamicPartitionPruning.enabled &#061; true&#xff08;3.0.0 \u8d77&#xff09;<\/td>\n<td>\u201cWhen true, we will generate predicate for partition column when it\u2019s used as join key\u201d<\/td>\n<td>\u751f\u6210\u5bf9\u5206\u533a\u5217\u7684\u8c13\u8bcd \u21d2 \u51cf\u626b\u63cf&#xff08;5.2.5&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.optimizer.runtime.bloomFilter.enabled &#061; true&#xff08;3.3.0 \u8d77&#xff09;<\/td>\n<td>\u201cif one side of a shuffle join has a selective predicate \u2026 we attempt to insert a bloom filter in the other side to reduce shuffle data\u201d<\/td>\n<td>\u5b98\u65b9\u63aa\u8f9e\u662f\u51cf shuffle \u6570\u636e\u91cf&#xff0c;\u4e0d\u662f\u51cf\u6253\u5f00\u7684\u6587\u4ef6\u6570<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u628a runtime bloom filter \u5f53\u6210&#034;\u6587\u4ef6\u88c1\u526a&#034;\u6765\u671f\u5f85&#xff0c;\u662f\u793e\u533a\u6587\u7ae0\u91cc\u6d41\u4f20\u6700\u5e7f\u7684\u8bef\u8bfb\u3002\u5b98\u65b9\u539f\u6587\u5199\u7684\u662f reduce shuffle data\u3002\u8981\u9a8c\u8bc1\u5b83\u5bf9\u626b\u63cf\u6709\u6ca1\u6709\u5e2e\u52a9&#xff0c;\u53ea\u770b files read \u6709\u6ca1\u6709\u53d8\u2014\u2014\u9ed8\u8ba4\u5b83\u4e0d\u4f1a\u53d8\u3002<\/p>\n<hr \/>\n<h3>5.4 \u6392\u5e8f\u3001\u805a\u7c07\u4e0e Z-Order&#xff1a;\u628a\u6570\u636e\u6446\u5230\u5c3a\u5b50\u4e0a<\/h3>\n<p>5.3 \u8bb2\u7684\u662f\u5c3a\u5b50&#xff08;\u7edf\u8ba1\u8fb9\u754c&#xff09;\u3002\u8fd9\u4e00\u8282\u8bb2\u600e\u4e48\u8ba9\u6570\u636e\u6446\u5f97&#034;\u6709\u8fb9\u754c&#034;\u3002\u5148\u7834\u4e00\u4e2a\u8bed\u6cd5\u5e7b\u89c9&#xff0c;\u518d\u8bb2\u9009\u578b&#xff0c;\u6700\u540e\u7ed9\u53ef\u8dd1\u7684\u4e09\u6bb5 SQL\u3002<\/p>\n<h4>5.4.1 Spark \u6ca1\u6709&#034;\u5199\u5165\u65f6\u6392\u5e8f&#034;\u7684 SQL \u5199\u6cd5<\/h4>\n<p>\u672c\u6b21\u53d6\u8bc1\u91cc\u6700\u53cd\u76f4\u89c9\u7684\u4e00\u6761&#xff1a;<\/p>\n<p>   \u5b98\u65b9 INSERT \u8bed\u6cd5\u9875&#xff08;sql-ref-syntax-dml-insert-into.html&#xff09;\u7684 Syntax \u5168\u6587&#xff1a;<br \/>\n   INSERT INTO [ TABLE ] table_identifier [ partition_spec ] [ ( column_list ) ]<br \/>\n       { VALUES ( &#8230; ) | query }<br \/>\n   \u2192 \u6574\u9875 0 \u6b21\u51fa\u73b0 ORDER BY \/ SORT BY \/ CLUSTER BY \/ DISTRIBUTE BY<\/p>\n<p>   \u5b98\u65b9 SELECT \u9875\u5bf9 ORDER BY \u7684\u539f\u6587\u53ea\u6709\u4e00\u53e5\u5b9a\u4f4d&#xff1a;<br \/>\n   &#034;Specifies an ordering of the rows of the complete result set of the query.<br \/>\n    The output rows are ordered across the partitions.&#034;<br \/>\n   \u2192 \u5b83\u63cf\u8ff0\u7684\u662f\u67e5\u8be2\u8f93\u51fa&#xff0c;\u4e0d\u662f\u843d\u76d8\u5e03\u5c40<\/p>\n<p>\u90a3&#034;\u6392\u5e8f\u5199\u5165&#034;\u5230\u5e95\u7531\u8c01\u51b3\u5b9a&#xff1f;\u5b98\u65b9\u628a\u5b83\u653e\u5728 DataSource V2 \u7684\u63a5\u53e3\u5c42&#xff0c;\u4e24\u6761\u539f\u6587\u8bf4\u5b8c\u4e86\u6574\u4ef6\u4e8b&#xff1a;<\/p>\n<p>\u201cA Write implementation can also implement RequiresDistributionAndOrdering to tell Spark how input data must be distributed and sorted before writing. Spark will insert shuffle and sort nodes as needed to satisfy these requirements.\u201d<\/p>\n<p>\u201cRequested distribution and ordering \u2014 Write connectors can request that Spark repartition and sort incoming data before writing, enabling optimized data layouts such as clustering or Z-ordering.\u201d<\/p>\n<p>\u7ed3\u8bba\u8bf4\u5f97\u76f4\u767d\u4e00\u70b9&#xff1a;\u5199\u5165\u5e03\u5c40\u7684\u6392\u5e8f\u80fd\u529b\u662f connector \u5411 Spark \u7533\u8bf7\u7684&#xff0c;\u4e0d\u662f\u7528\u6237\u5728 SQL \u91cc\u5199 ORDER BY \u6362\u6765\u7684\u3002 \u6240\u4ee5\u672c\u7ae0\u540e\u9762\u6240\u6709&#034;\u600e\u4e48\u6392&#034;\u7684\u8bed\u6cd5\u90fd\u6765\u81ea\u8868\u683c\u5f0f\u81ea\u5df1\u7684\u6269\u5c55&#xff0c;\u800c\u4e0d\u662f Spark SQL&#xff1a;<\/p>\n<ul>\n<li>Iceberg \u63d0\u4f9b SQL \u6269\u5c55\u6a21\u5757&#xff08;\u5b98\u65b9\u539f\u6587&#xff1a;Iceberg 0.11.0 \u8d77\u4e3a Spark \u589e\u52a0\u6269\u5c55\u547d\u4ee4&#xff0c;\u5982 CALL \u4e0e ALTER TABLE &#8230; WRITE ORDERED BY&#xff09;&#xff0c;\u6709 WRITE ORDERED BY \/ WRITE LOCALLY ORDERED BY \/ WRITE UNORDERED \/ WRITE DISTRIBUTED BY PARTITION \u56db\u6761\u3002<\/li>\n<li>Paimon \u628a\u5e03\u5c40\u4ea4\u7ed9\u7ef4\u62a4\u8fc7\u7a0b&#xff1a;append \u8868\u7684 incremental clustering&#xff0c;\u4ee5\u53ca CALL sys.compact(&#8230; order_strategy &#061;&gt; &#039;zorder&#039;, order_by &#061;&gt; &#8230;)\u3002<\/li>\n<li>Hive \u98ce\u683c\u8868\u7528 DDL \u5b50\u53e5 CLUSTERED BY &#8230; SORTED BY &#8230; INTO n BUCKETS&#xff08;\u5b98\u65b9\u539f\u6587&#xff1a;\u201cBucketing is an optimization technique that uses buckets (and bucketing columns) to determine data partitioning and avoid data shuffle.\u201d&#xff09;&#xff0c;\u89c1 5.5\u3002<\/li>\n<\/ul>\n<p>\u8868 5-4 \u4e94\u79cd&#034;\u6392\u5e8f&#034;\u80fd\u529b\u5bf9\u7167&#xff08;\u5b98\u65b9\u5b9a\u4f4d\u9010\u5b57\u6458&#xff09;<\/p>\n<table>\n<tr>\u80fd\u529b\u5c5e\u4e8e\u54ea\u4e00\u4fa7\u5b98\u65b9\u539f\u6587\u8981\u70b9\u6539\u53d8\u4ec0\u4e48\u5f71\u54cd\u4e0b\u6e38\u67e5\u8be2<\/tr>\n<tbody>\n<tr>\n<td>SELECT &#8230; ORDER BY<\/td>\n<td>Spark \u67e5\u8be2<\/td>\n<td>\u201cordered across the partitions\u201d&#xff0c;\u4e0e SORT BY\/CLUSTER BY\/DISTRIBUTE BY \u4e92\u65a5<\/td>\n<td>\u7ed3\u679c\u8f93\u51fa\u987a\u5e8f<\/td>\n<td>\u5426<\/td>\n<\/tr>\n<tr>\n<td>SELECT &#8230; CLUSTER BY<\/td>\n<td>Spark \u67e5\u8be2<\/td>\n<td>\u201conly ensures that the resultant rows are sorted within each partition and does not guarantee a total order of output\u201d<\/td>\n<td>\u7ed3\u679c\u5206\u533a\u5185\u987a\u5e8f<\/td>\n<td>\u5426<\/td>\n<\/tr>\n<tr>\n<td>RequiresDistributionAndOrdering<\/td>\n<td>DSv2 \u5199\u63a5\u53e3<\/td>\n<td>connector \u58f0\u660e\u9700\u6c42&#xff0c;\u201cSpark will insert shuffle and sort nodes as needed\u201d<\/td>\n<td>\u5199\u5165\u5e03\u5c40<\/td>\n<td>\u662f<\/td>\n<\/tr>\n<tr>\n<td>WRITE ORDERED BY \/ LOCALLY ORDERED BY<\/td>\n<td>Iceberg \u6269\u5c55<\/td>\n<td>\u201cconfigured with a sort order that is used to automatically sort data that is written to the table in some engines. For example, MERGE INTO in Spark will use the table ordering\u201d<\/td>\n<td>\u8868\u9ed8\u8ba4\u5e03\u5c40 &#043; \u65b0\u5199\u5165<\/td>\n<td>\u662f&#xff08;\u53ea\u5bf9\u65b0\u5199\u5165\u548c MERGE \u751f\u6548&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>CLUSTERED BY &#8230; SORTED BY &#8230; INTO n BUCKETS<\/td>\n<td>Spark DDL<\/td>\n<td>\u201cdetermine data partitioning and avoid data shuffle\u201d<\/td>\n<td>\u6876\u5f52\u5c5e &#043; \u6876\u5185\u5e8f<\/td>\n<td>\u662f&#xff0c;\u4e14\u5e26\u6765\u514d shuffle<\/td>\n<\/tr>\n<tr>\n<td>Paimon clustering.strategy \/ Sort Compact<\/td>\n<td>Paimon \u7ef4\u62a4\u8fc7\u7a0b<\/td>\n<td>\u201cimproves the data layout of append tables by sorting selected files on frequently filtered columns\u201d&#xff1b;\u201cdoes not replace SQL ORDER BY\u201d<\/td>\n<td>\u5b58\u91cf\u6587\u4ef6\u5e03\u5c40<\/td>\n<td>\u662f<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u6700\u540e\u4e00\u5217\u90a3\u53e5\u8bdd\u503c\u5f97\u5355\u72ec\u6284\u8fdb\u8bc4\u5ba1\u6e05\u5355&#xff1a;WRITE ORDERED BY \u6539\u7684\u662f&#034;\u4ee5\u540e\u600e\u4e48\u5199&#034;&#xff0c;\u4e0d\u662f&#034;\u5df2\u7ecf\u5199\u597d\u7684\u600e\u4e48\u6446&#034;\u3002 \u5b58\u91cf\u6570\u636e\u53ea\u6709 rewrite \u4e00\u6761\u8def\u3002\u751f\u4ea7\u4e0a\u6700\u5e38\u89c1\u7684\u9519\u6cd5\u662f&#034;\u7ed9\u8868\u52a0\u4e86\u6392\u5e8f&#xff0c;\u7136\u540e\u53d1\u73b0\u67e5\u8be2\u6ca1\u53d8\u5feb&#034;\u2014\u2014\u987a\u5e8f\u5168\u5bf9&#xff0c;\u53ea\u662f\u6ca1\u4eba\u91cd\u5199\u5386\u53f2\u6587\u4ef6\u3002<\/p>\n<h4>5.4.2 \u7ebf\u6027 \/ Z-Order \/ Hilbert&#xff1a;\u6309\u67e5\u8be2\u5f62\u6001\u9009&#xff0c;\u4e0d\u6309\u540d\u6c14\u9009<\/h4>\n<p>\u4e24\u4e2a\u7ef4\u5ea6&#xff08;\u6a2a\u8f74 city_id\u3001\u7eb5\u8f74 report_date&#xff09;\u7684\u4e24\u79cd\u6446\u6cd5&#xff1a;<\/p>\n<p>   \u7ebf\u6027\u6392\u5e8f sort(city_id, report_date)<br \/>\n   \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510        city \u5148\u5206\u7ec4&#xff0c;\u7ec4\u5185\u6309\u65e5\u671f\u8fde\u7eed<br \/>\n   \u2502 \u2588\u2588\u2502\u2593\u2593\u2502\u2591\u2591\u2502\u2500\u2500\u2502\u2500\u2500                   \u2502        \u00b7 \u67e5 city&#061;?           \u21d2 1 \u6bb5\u8fde\u7eed\u5757&#xff0c;\u6781\u597d<br \/>\n   \u2502 \u2588\u2588\u2502\u2593\u2593\u2502\u2591\u2591\u2502\u2500\u2500\u2502\u2500\u2500                   \u2502        \u00b7 \u67e5 date&#061;?           \u21d2 \u6bcf\u6bb5\u5404\u6cbe\u4e00\u70b9&#xff0c;\u51e0\u4e4e\u5168\u8bfb<br \/>\n   \u2502 \u2588\u2588\u2502\u2593\u2593\u2502\u2591\u2591\u2502\u2500\u2500\u2502\u2500\u2500                   \u2502        \u00b7 \u67e5 city&#061;? AND date&#061;? \u21d2 1 \u5757\u5185\u5b9a\u4f4d&#xff0c;\u6781\u597d<br \/>\n   \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518        \u00b7 \u4f18\u70b9&#xff1a;\u5355\u7ef4\u9009\u62e9\u6027\u6700\u5f3a&#xff0c;\u4ee3\u4ef7\u6700\u5c0f<br \/>\n     \u2588\u2588&#061;city1  \u2593\u2593&#061;city2  \u2591\u2591&#061;city3<\/p>\n<p>   Z-Order&#xff08;city \u4e0e date \u7684\u6bd4\u7279\u4f4d\u4ea4\u9519&#xff09;<br \/>\n   \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510        \u7a7a\u95f4\u4e0a\u76f8\u90bb\u7684\u683c\u5b50\u5728\u7f16\u7801\u4e0a\u4e5f\u76f8\u90bb<br \/>\n   \u2502 A\u2502B\u2502A\u2502C\u2502B\u2502A\u2502C\u2502B                  \u2502        \u00b7 \u67e5 city&#061;?           \u21d2 \u82e5\u5e72\u5757&#xff0c;\u80fd\u88c1<br \/>\n   \u2502 \u2500\u253c\u2500\u253c\u2500\u253c\u2500\u253c\u2500\u253c\u2500\u253c\u2500\u253c\u2500                  \u2502        \u00b7 \u67e5 date&#061;?           \u21d2 \u82e5\u5e72\u5757&#xff0c;\u80fd\u88c1<br \/>\n   \u2502 D\u2502B\u2502D\u2502C\u2502B\u2502D\u2502C\u2502A                  \u2502        \u00b7 \u67e5 city&#061;? AND date&#061;? \u21d2 1~2 \u5757&#xff0c;\u597d<br \/>\n   \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518        \u00b7 \u4ee3\u4ef7&#xff1a;\u6ca1\u6709\u4efb\u4f55\u4e00\u7ef4\u62ff\u5230&#034;\u8fde\u7eed\u6574\u6bb5&#034;<br \/>\n     \u540c\u4e00\u5b57\u6bcd &#061; \u7f16\u7801\u4e0a\u76f8\u90bb\u7684\u5757                  \u591a\u7ef4\u5404\u81ea\u90fd\u53ea\u62ff\u5230&#034;\u90e8\u5206\u6536\u76ca&#034;<\/p>\n<p>   Hilbert&#xff1a;\u4e0e Z-Order \u540c\u65cf&#xff0c;\u5c40\u90e8\u6027\u4fdd\u6301\u66f4\u597d&#xff0c;\u7ef4\u5ea6\u591a\u65f6\u4f18\u52bf\u663e\u73b0<\/p>\n<p>\u8868 5-5 \u4e09\u79cd\u5e03\u5c40\u5bf9\u4e0d\u540c\u67e5\u8be2\u5f62\u6001\u7684\u8868\u73b0&#xff08;\u7b14\u8005\u5f52\u7eb3&#xff0c;\u5224\u636e\u89c1 5.6 \u5b9e\u6d4b&#xff09;<\/p>\n<table>\n<tr>\u67e5\u8be2\u5f62\u6001\u7ebf\u6027 sort(a,b)Z-Order(a,b)Hilbert(a,b)<\/tr>\n<tbody>\n<tr>\n<td>a &#061; \u5e38\u91cf<\/td>\n<td>\u6700\u4f18&#xff08;\u8fde\u7eed\u533a\u95f4&#xff09;<\/td>\n<td>\u597d&#xff08;\u82e5\u5e72\u5757&#xff09;<\/td>\n<td>\u597d<\/td>\n<\/tr>\n<tr>\n<td>b &#061; \u5e38\u91cf<\/td>\n<td>\u5dee&#xff08;\u6563\u5e03\u5168\u90e8\u5757&#xff09;<\/td>\n<td>\u597d<\/td>\n<td>\u597d<\/td>\n<\/tr>\n<tr>\n<td>a &#061; \u5e38\u91cf AND b &#061; \u5e38\u91cf<\/td>\n<td>\u6700\u4f18<\/td>\n<td>\u597d<\/td>\n<td>\u597d<\/td>\n<\/tr>\n<tr>\n<td>a BETWEEN \u2026 AND b BETWEEN \u2026<\/td>\n<td>\u53ea\u5bf9\u7b2c\u4e00\u5217\u53cb\u597d<\/td>\n<td>\u4e2d<\/td>\n<td>\u8f83\u597d<\/td>\n<\/tr>\n<tr>\n<td>\u7ef4\u5ea6\u6570 \u2265 5<\/td>\n<td>\u2014<\/td>\n<td>\u4ea4\u9519\u4f4d\u6570\u81a8\u80c0&#xff0c;\u6536\u76ca\u4e0b\u964d<\/td>\n<td>\u5b98\u65b9\u5efa\u8bae\u5728\u6b64\u5207\u6362<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u5b98\u65b9\u7ed9\u51fa\u7684\u9009\u578b\u89c4\u5219&#xff08;\u6bd4\u4efb\u4f55\u53e3\u8bc0\u90fd\u53ef\u4fe1&#xff09;&#xff1a;Paimon \u7684 clustering.strategy \u9ed8\u8ba4 auto&#xff0c;\u5b98\u65b9\u9010\u5b57\u8bf4\u660e\u662f \u201cAutomatic selection uses order for one column, zorder for two to four, and hilbert for five or more.\u201d \u2014\u2014 1 \u5217\u7ebf\u6027\u30012~4 \u5217 Z-Order\u30015 \u5217\u4ee5\u4e0a Hilbert\u3002Iceberg \u4fa7\u5219\u5728 rewrite_data_files \u7684 sort_order \u91cc\u540c\u65f6\u63d0\u4f9b zorder(c1,c2,c3) \u4e0e hilbert(c1,c2,c3) \u4e24\u79cd\u5199\u6cd5\u3002\u8d85\u8fc7\u56db\u5217\u8fd8\u5728\u5f80 Z-Order \u91cc\u585e&#xff0c;\u5c31\u5df2\u7ecf\u8d8a\u51fa\u5b98\u65b9\u5efa\u8bae\u533a\u95f4\u4e86\u3002<\/p>\n<p>\u843d\u5230\u8f66\u8054\u7f51\u8fd9\u6761\u94fe\u8def\u4e0a&#xff0c;\u4e24\u4e2a\u771f\u5b9e\u9700\u6c42\u7ed9\u51fa\u4e24\u4e2a\u76f8\u53cd\u7684\u7b54\u6848&#xff1a;<\/p>\n<p>\u9700\u6c42 A&#xff1a;\u8bca\u65ad\/\u5ba2\u670d&#034;\u67e5\u4e00\u53f0\u8f66\u67d0\u4e00\u5929\u7684\u8f68\u8ff9&#034;<br \/>\n    \u67e5\u8be2\u5f62\u72b6&#xff1a;vin &#061; ? AND report_date &#061; ?        &#xff08;\u4e24\u5217\u90fd\u662f\u7b49\u503c&#xff0c;\u4e14\u51e0\u4e4e\u603b\u662f\u540c\u65f6\u51fa\u73b0&#xff09;<br \/>\n    \u9009\u62e9&#xff1a;\u7ebf\u6027 WRITE ORDERED BY (vin, report_date)<br \/>\n    \u7406\u7531&#xff1a;\u7ebf\u6027\u5bf9&#034;\u590d\u5408\u7b49\u503c &#043; \u7ec4\u5185\u8303\u56f4&#034;\u662f\u6700\u4f18\u5f62\u72b6&#xff1b;\u6362 Z-Order \u4f1a\u8ba9\u540c\u4e00 vin \u7684\u6570\u636e<br \/>\n          \u4e0d\u518d\u8fde\u7eed&#xff0c;\u53cd\u800c\u727a\u7272\u4e86&#034;\u6309\u8f66\u6574\u6bb5\u5bfc\u51fa&#034;\u8fd9\u7c7b\u4f5c\u4e1a<\/p>\n<p>\u9700\u6c42 B&#xff1a;\u57ce\u5e02\u8fd0\u8425\u770b\u677f&#034;\u4efb\u610f\u57ce\u5e02 \u00d7 \u4efb\u610f\u65e5\u671f\u533a\u95f4&#034;<br \/>\n    \u67e5\u8be2\u5f62\u72b6&#xff1a;city_id \u53ef\u80fd\u6709\u3001date \u8303\u56f4\u51e0\u4e4e\u603b\u6709&#xff0c;\u4e14 city \u5e38\u5e38\u5355\u72ec\u51fa\u73b0<br \/>\n    \u9009\u62e9&#xff1a;\u5206\u533a\u88c1\u65e5\u671f &#043; Z-Order(city_id, report_date) \u6574\u7406\u5757\u5185\u76f8\u5173\u6027<br \/>\n    \u7406\u7531&#xff1a;city \u4e0d\u5728\u5206\u533a\u952e\u91cc&#xff0c;\u7ebf\u6027\u6392\u5e8f\u5bf9&#034;\u53ea\u7ed9 city&#034;\u96f6\u6536\u76ca&#xff1b;Z-Order \u662f\u552f\u4e00\u80fd\u540c\u65f6<br \/>\n          \u7167\u987e\u4e24\u7ef4\u7684\u624b\u6bb5<\/p>\n<p>\u9700\u6c42 C&#xff1a;\u628a\u4e24\u8005\u90fd\u60f3\u8981\u7684\u8868<br \/>\n    \u7b54\u6848&#xff1a;\u4e0d\u5b58\u5728\u4e00\u4e2a\u5e03\u5c40\u80fd\u540c\u65f6\u6700\u4f18\u3002\u8fd9\u65f6\u8be5\u62c6\u8868&#xff08;DWD \u660e\u7ec6\u6309 vin \u6392\u3001DWS \u57ce\u5e02\u6c47\u603b<br \/>\n          \u6309 city \u6392&#xff09;&#xff0c;\u4e0d\u662f\u7ee7\u7eed\u8c03\u6392\u5e8f\u53c2\u6570\u3002\u8fd9\u662f\u672c\u7ae0\u5bf9&#034;\u8fc7\u5ea6\u4f18\u5316&#034;\u6700\u91cd\u8981\u7684\u4e00\u6761\u529d\u963b\u3002<\/p>\n<p>\u4e24\u6761\u5fc5\u987b\u4e00\u8d77\u8bf4\u51fa\u6765\u7684\u4ee3\u4ef7&#xff0c;\u5426\u5219\u8bfb\u8005\u4f1a\u4ee5\u4e3a Z-Order \u514d\u8d39&#xff1a;<\/p>\n<li>Z-Order \u7684\u7d27\u51d1\u6027\u4f1a\u88ab\u65b0\u6570\u636e\u7a00\u91ca\u3002 \u4e4b\u540e\u8ffd\u52a0\u5199\u5165\u7684\u6570\u636e\u4e0d\u53c2\u4e0e\u65e2\u6709\u7684\u4ea4\u9519\u7f16\u7801&#xff0c;\u5757\u8fb9\u754c\u8d8a\u6765\u8d8a\u5bbd\u2014\u2014\u5fc5\u987b\u5468\u671f\u6027 rewrite\u3002\u8fd9\u5c31\u628a\u5b83\u4ece&#034;\u4e00\u6b21 DDL \u51b3\u5b9a&#034;\u53d8\u6210\u4e86&#034;\u4e00\u9879\u957f\u671f\u7ef4\u62a4\u6210\u672c&#034;&#xff0c;\u5f52\u5c5e\u7b2c 10 \u7ae0\u3002<\/li>\n<li>\u5b83\u4f1a\u4f24\u5bb3\u6709\u5e8f\u6027\u672c\u8eab\u3002 \u4ea4\u9519\u7f16\u7801\u4e4b\u540e&#xff0c;\u6309 vin \u987a\u5e8f\u6d41\u5f0f\u5bfc\u51fa\u3001\u6309 vin \u505a window&#xff08;\u7b2c 13 \u7ae0\u884c\u7a0b\u6bb5\u5207\u5206&#xff09;\u8fd9\u7c7b\u4f5c\u4e1a&#xff0c;\u4ece&#034;\u6570\u636e\u5df2\u6709\u5e8f&#034;\u9000\u5316\u4e3a&#034;\u5fc5\u987b\u91cd\u6392&#034;\u3002\u7b2c 4 \u7ae0 4.5.2 \u8bf4\u8fc7 Sort \u662f\u6700\u5e38\u88ab\u767d\u767d\u4ed8\u6389\u7684\u5f00\u9500\u2014\u2014\u8fd9\u91cc\u5b83\u4e0d\u662f\u767d\u767d\u4ed8&#xff0c;\u662f\u88ab\u5e03\u5c40\u903c\u7740\u4ed8\u3002<\/li>\n<h4>5.4.3 \u5b9e\u64cd&#xff1a;\u4ece\u5efa\u8868\u5230\u91cd\u5199\u5386\u53f2\u6570\u636e<\/h4>\n<p><span class=\"token comment\">&#8212; \u6587\u4ef6&#xff1a;zorder_layout.sql<\/span><br \/>\n<span class=\"token comment\">&#8212; \u2460 \u5efa DWD \u660e\u7ec6\u8868&#xff1a;\u5206\u533a\u4e0e\u6392\u5e8f\u5206\u5f00\u5199&#xff0c;\u4e24\u4ef6\u4e8b\u5404\u7ba1\u4e00\u6bb5<\/span><br \/>\n<span class=\"token keyword\">CREATE<\/span> <span class=\"token keyword\">TABLE<\/span> <span class=\"token keyword\">IF<\/span> <span class=\"token operator\">NOT<\/span> <span class=\"token keyword\">EXISTS<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>dwd_vehicle_telemetry <span class=\"token punctuation\">(<\/span><br \/>\n    vin            STRING<span class=\"token punctuation\">,<\/span><br \/>\n    city_id        <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    report_date    <span class=\"token keyword\">DATE<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    reported_at    <span class=\"token keyword\">TIMESTAMP<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    speed          <span class=\"token keyword\">FLOAT<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    battery        <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    battery_pct    <span class=\"token keyword\">TINYINT<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    vehicle_status STRING<span class=\"token punctuation\">,<\/span><br \/>\n    seq            <span class=\"token keyword\">BIGINT<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    __dt__         STRING<br \/>\n<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">USING<\/span> iceberg<br \/>\nPARTITIONED <span class=\"token keyword\">BY<\/span> <span class=\"token punctuation\">(<\/span>days<span class=\"token punctuation\">(<\/span>report_date<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\nTBLPROPERTIES <span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#039;format-version&#039;<\/span>                  <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;2&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string\">&#039;write.target-file-size-bytes&#039;<\/span>    <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;33554432&#039;<\/span><span class=\"token punctuation\">,<\/span>    <span class=\"token comment\">&#8212; 32MB&#xff1a;\u5b9e\u9a8c\u89c4\u6a21\u5fc5\u987b\u5207\u5c0f&#xff0c;\u7406\u7531\u89c1\u4e0b<\/span><br \/>\n    <span class=\"token string\">&#039;write.parquet.row-group-size-bytes&#039;<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;4194304&#039;<\/span>   <span class=\"token comment\">&#8212; 4MB&#xff1a;\u8ba9 row-group \u7ea7\u88c1\u526a\u53ef\u89c2\u6d4b<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u2461 \u8868\u7ea7\u5168\u5c40\u6392\u5e8f&#xff08;\u5bf9&#034;\u65b0\u5199\u5165&#034;\u4e0e&#034;MERGE INTO&#034;\u751f\u6548&#xff09;<\/span><br \/>\n<span class=\"token keyword\">ALTER<\/span> <span class=\"token keyword\">TABLE<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>dwd_vehicle_telemetry<br \/>\n    <span class=\"token keyword\">WRITE<\/span> ORDERED <span class=\"token keyword\">BY<\/span> city_id<span class=\"token punctuation\">,<\/span> report_date<span class=\"token punctuation\">,<\/span> vin<span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u2462 \u6216\u8005&#xff1a;task \u5185\u5c40\u90e8\u6392\u5e8f&#xff0c;\u7701\u6389\u4e00\u6b21\u5168\u5c40 shuffle&#xff08;\u2461\u2462\u4e92\u65a5&#xff0c;\u5b9e\u9645\u53ea\u4fdd\u7559\u4e00\u6761&#xff09;<\/span><br \/>\n<span class=\"token keyword\">ALTER<\/span> <span class=\"token keyword\">TABLE<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>dwd_vehicle_telemetry<br \/>\n    <span class=\"token keyword\">WRITE<\/span> LOCALLY ORDERED <span class=\"token keyword\">BY<\/span> city_id<span class=\"token punctuation\">,<\/span> vin<span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u2463 \u53d6\u6d88\u8868\u7ea7\u6392\u5e8f<\/span><br \/>\n<span class=\"token keyword\">ALTER<\/span> <span class=\"token keyword\">TABLE<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>dwd_vehicle_telemetry <span class=\"token keyword\">WRITE<\/span> UNORDERED<span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u2464 \u5199\u5165\u5386\u53f2\u6570\u636e&#xff08;\u7b2c 2 \u7ae0\u6570\u636e\u96c6\u6574\u6bb5\u642c\u5165&#xff0c;\u4fdd\u6301\u53ef\u590d\u73b0&#xff09;<\/span><br \/>\n<span class=\"token keyword\">INSERT<\/span> <span class=\"token keyword\">INTO<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>dwd_vehicle_telemetry<br \/>\n<span class=\"token keyword\">SELECT<\/span> vin<span class=\"token punctuation\">,<\/span> city_id<span class=\"token punctuation\">,<\/span> report_date<span class=\"token punctuation\">,<\/span> reported_at<span class=\"token punctuation\">,<\/span> speed<span class=\"token punctuation\">,<\/span> battery<span class=\"token punctuation\">,<\/span> battery_pct<span class=\"token punctuation\">,<\/span><br \/>\n       vehicle_status<span class=\"token punctuation\">,<\/span> seq<span class=\"token punctuation\">,<\/span> __dt__<br \/>\n<span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<br \/>\n<span class=\"token keyword\">WHERE<\/span> __dt__ <span class=\"token operator\">BETWEEN<\/span> <span class=\"token string\">&#039;2026-10-01&#039;<\/span> <span class=\"token operator\">AND<\/span> <span class=\"token string\">&#039;2026-10-30&#039;<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e<\/p>\n<table>\n<tr>\u9879\u53d6\u503c\u4e3a\u4ec0\u4e48\u8fd9\u4e48\u5199<\/tr>\n<tbody>\n<tr>\n<td>PARTITIONED BY (days(report_date))<\/td>\n<td>\u9690\u85cf\u5206\u533a&#xff0c;\u6309\u5929<\/td>\n<td>\u7528\u6237\u67e5\u8be2\u5199 report_date&#xff0c;\u4e0d\u5199\u5206\u533a\u5b57\u6bb5&#xff08;5.1 \u7684\u6838\u5fc3\u5dee\u522b&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>write.target-file-size-bytes<\/td>\n<td>32MB&#xff08;\u5b98\u65b9\u9ed8\u8ba4 512MB&#xff09;<\/td>\n<td>\u672c\u5b9e\u9a8c\u603b\u6570\u636e\u53ea\u6709 20 \u4e07\u884c\u3001\u7ea6 15~25MB\u3002\u7528\u9ed8\u8ba4\u503c\u53ea\u4f1a\u4ea7\u51fa 1 \u4e2a\u6587\u4ef6&#xff0c;1 \u4e2a\u6587\u4ef6\u4e0a\u4efb\u4f55\u6392\u5e8f\u4e0e\u7edf\u8ba1\u90fd\u6d4b\u4e0d\u51fa\u88c1\u526a<\/td>\n<\/tr>\n<tr>\n<td>write.parquet.row-group-size-bytes<\/td>\n<td>4MB&#xff08;\u5b98\u65b9\u9ed8\u8ba4 128MB&#xff09;<\/td>\n<td>\u540c\u4e0a&#xff1a;\u8ba9&#034;\u540c\u4e00\u6587\u4ef6\u5185\u8df3\u8fc7 row-group&#034;\u8fd9\u4ef6\u4e8b\u5728\u5c0f\u6570\u636e\u4e0a\u4ecd\u7136\u53d1\u751f\u3002\u751f\u4ea7\u65b9\u5411\u662f\u76f8\u53cd\u7684&#xff1a;\u5408\u5e76\u3001\u653e\u5927<\/td>\n<\/tr>\n<tr>\n<td>WRITE ORDERED BY city_id, report_date, vin<\/td>\n<td>\u4e09\u5217\u7ebf\u6027<\/td>\n<td>\u5217\u5e8f\u5373\u6536\u76ca\u5e8f&#xff1a;\u6700\u7c97\u7684\u9009\u62e9\u6027\u653e\u524d\u9762\u3002Z-Order \u7248\u672c\u89c1\u4e0b\u4e00\u6b65\u7684 rewrite<\/td>\n<\/tr>\n<tr>\n<td>WRITE LOCALLY ORDERED BY<\/td>\n<td>\u5bf9\u7167\u7ec4<\/td>\n<td>\u5b98\u65b9\u660e\u786e&#034;to order within each task, not across tasks&#034;\u2014\u2014\u5b83\u4e0d\u505a\u5168\u5c40 shuffle&#xff0c;\u5757\u5185\u76f8\u5173\u6027\u5dee&#xff0c;\u4f46\u5199\u5165\u4fbf\u5b9c\u5f97\u591a<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u8f93\u51fa\u8868\u7ed3\u6784&#xff1a;\u4e0e\u6e90\u5217\u4e00\u81f4&#xff08;10 \u5217&#xff09;\u3002report_date \u662f DATE&#xff0c;\u9690\u85cf\u5206\u533a\u5b57\u6bb5 report_date_day \u4e0d\u51fa\u73b0\u5728 schema \u91cc\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6<\/p>\n<p><span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> rows_total<span class=\"token punctuation\">,<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token keyword\">DISTINCT<\/span> vin<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> vin_cnt<span class=\"token punctuation\">,<\/span><br \/>\n       <span class=\"token function\">min<\/span><span class=\"token punctuation\">(<\/span>report_date<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> dmin<span class=\"token punctuation\">,<\/span> <span class=\"token function\">max<\/span><span class=\"token punctuation\">(<\/span>report_date<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> dmax<br \/>\n<span class=\"token keyword\">FROM<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>dwd_vehicle_telemetry<span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u671f\u671b&#xff1a;200000 \/ 200000 \/ 2026-10-01 \/ 2026-10-30&#xff0c;\u4e0e\u7b2c 2 \u7ae0 ODS \u5b8c\u5168\u4e00\u81f4<\/span><\/p>\n<p><span class=\"token keyword\">SELECT<\/span> file_format<span class=\"token punctuation\">,<\/span> record_count<span class=\"token punctuation\">,<\/span> file_size_in_bytes<br \/>\n<span class=\"token keyword\">FROM<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>dwd_vehicle_telemetry<span class=\"token punctuation\">.<\/span>files<br \/>\n<span class=\"token keyword\">ORDER<\/span> <span class=\"token keyword\">BY<\/span> file_size_in_bytes <span class=\"token keyword\">DESC<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; Spark \u4fa7\u5143\u6570\u636e\u8868\u7528\u70b9\u53f7 .files&#xff08;\u524d\u4f5c\u786c\u7ea6\u675f&#xff1a;Flink \u4fa7\u8981\u5199 $files&#xff0c;\u4e24\u8005\u4e0d\u53ef\u6df7\u7528&#xff09;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u671f\u671b&#xff1a;\u6587\u4ef6\u6570 \u2265 2&#xff1b;\u6bcf\u4e2a\u6587\u4ef6 record_count \u4e0e file_size_in_bytes \u76f8\u4e92\u5339\u914d&#xff08;\u65e0\u6781\u7aef\u957f\u5c3e\u6587\u4ef6&#xff09;<\/span><\/p>\n<p>\u5386\u53f2\u6570\u636e\u7684\u5e03\u5c40\u53ea\u80fd\u91cd\u5199\u3002rewrite_data_files \u5728 Iceberg 1.12.0 \u7684\u53c2\u6570\u4e0e\u65e7\u7248\u5dee\u522b\u5f88\u5927&#xff0c;\u8fd9\u662f\u672c\u7ae0\u6700\u9700\u8981&#034;\u6309\u7248\u672c\u6838\u5bf9&#034;\u7684\u4e00\u6bb5&#xff1a;<\/p>\n<p><span class=\"token comment\">&#8212; \u2465 \u5bf9\u5386\u53f2\u6570\u636e\u505a Z-Order \u91cd\u6392&#xff08;1.12 \u7684\u53c2\u6570\u540d&#xff09;<\/span><br \/>\n<span class=\"token keyword\">CALL<\/span> iceberg<span class=\"token punctuation\">.<\/span>system<span class=\"token punctuation\">.<\/span>rewrite_data_files<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token keyword\">table<\/span>    <span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token string\">&#039;demo.dwd_vehicle_telemetry&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    strategy <span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token string\">&#039;sort&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    sort_order <span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token string\">&#039;zorder(city_id, report_date)&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token keyword\">where<\/span>    <span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token string\">&#034;report_date &gt;&#061; &#039;2026-10-01&#039;&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    options  <span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&gt;<\/span> map<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;target-file-size-bytes&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;8388608&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                    <span class=\"token string\">&#039;max-concurrent-file-group-rewrites&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;5&#039;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u2466 \u53ea\u5408\u5e76\u5c0f\u6587\u4ef6&#xff0c;\u4e0d\u52a8\u987a\u5e8f<\/span><br \/>\n<span class=\"token keyword\">CALL<\/span> iceberg<span class=\"token punctuation\">.<\/span>system<span class=\"token punctuation\">.<\/span>rewrite_data_files<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token keyword\">table<\/span> <span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token string\">&#039;demo.dwd_vehicle_telemetry&#039;<\/span><span class=\"token punctuation\">,<\/span> strategy <span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token string\">&#039;binpack&#039;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u2467 \u770b\u7ed3\u679c&#xff1a;\u6587\u4ef6\u6570\u4e0e\u6bcf\u6587\u4ef6\u884c\u6570<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> file_cnt<span class=\"token punctuation\">,<\/span> <span class=\"token function\">sum<\/span><span class=\"token punctuation\">(<\/span>record_count<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> rows_total<span class=\"token punctuation\">,<\/span><br \/>\n       <span class=\"token function\">avg<\/span><span class=\"token punctuation\">(<\/span>file_size_in_bytes<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> avg_bytes<br \/>\n<span class=\"token keyword\">FROM<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>dwd_vehicle_telemetry<span class=\"token punctuation\">.<\/span>files<span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u9a8c\u6536&#xff1a;rows_total \u5fc5\u987b\u4e0e\u2464\u524d\u540e\u4e00\u81f4&#xff08;rewrite \u4e0d\u5220\u884c&#xff09;&#xff1b;file_cnt \u53d8\u5316\u7b26\u5408 target \u8bbe\u5b9a<\/span><\/p>\n<p>\u8868 5-6 rewrite_data_files \u53c2\u6570\u4e0e\u5b98\u65b9\u9ed8\u8ba4\u503c&#xff08;Iceberg 1.12.0&#xff0c;\u9010\u5b57&#xff09;<\/p>\n<table>\n<tr>\u53c2\u6570\u5fc5\u586b\u5b98\u65b9\u9ed8\u8ba4\u539f\u6587\u8981\u70b9<\/tr>\n<tbody>\n<tr>\n<td>table<\/td>\n<td>\u2714<\/td>\n<td>\u2014<\/td>\n<td>\u8868\u540d<\/td>\n<\/tr>\n<tr>\n<td>strategy<\/td>\n<td><\/td>\n<td>binpack<\/td>\n<td>\u201cName of the strategy &#8211; binpack or sort. Defaults to binpack strategy\u201d<\/td>\n<\/tr>\n<tr>\n<td>sort_order<\/td>\n<td><\/td>\n<td>\u8868\u7684 sort order<\/td>\n<td>\u201cFor Zorder use a comma separated list of columns within zorder(). Example: zorder(c1,c2,c3). For Hilbert use \u2026 hilbert(c1,c2,c3). Else, Comma separated sort orders in the format (ColumnName SortDirection NullOrder)\u201d<\/td>\n<\/tr>\n<tr>\n<td>where<\/td>\n<td><\/td>\n<td>\u672a\u8bbe<\/td>\n<td>\u201cpredicate as a string used for filtering the files. Note that all files that may contain data matching the filter will be selected for rewriting\u201d<\/td>\n<\/tr>\n<tr>\n<td>options<\/td>\n<td><\/td>\n<td>\u672a\u8bbe<\/td>\n<td>map&#xff0c;\u4f20\u7ed9\u5e95\u5c42\u52a8\u4f5c\u7684\u5199\u9009\u9879<\/td>\n<\/tr>\n<tr>\n<td>target-file-size-bytes<\/td>\n<td><\/td>\n<td>536870912&#xff08;512MB&#xff09;<\/td>\n<td>\u76ee\u6807\u5927\u5c0f<\/td>\n<\/tr>\n<tr>\n<td>min-file-size-bytes \/ max-file-size-bytes<\/td>\n<td><\/td>\n<td>75% \/ 180% of target<\/td>\n<td>\u51b3\u5b9a&#034;\u54ea\u4e9b\u6587\u4ef6\u7b97\u5c0f\/\u5927&#034;<\/td>\n<\/tr>\n<tr>\n<td>min-input-files<\/td>\n<td><\/td>\n<td>5<\/td>\n<td>\u4e00\u7ec4\u81f3\u5c11\u51e0\u4e2a\u6587\u4ef6\u624d\u91cd\u5199<\/td>\n<\/tr>\n<tr>\n<td>rewrite-all<\/td>\n<td><\/td>\n<td>false<\/td>\n<td>\u9ed8\u8ba4\u4e0d\u52a8&#034;\u5df2\u7ecf\u5408\u683c&#034;\u7684\u6587\u4ef6<\/td>\n<\/tr>\n<tr>\n<td>max-concurrent-file-group-rewrites<\/td>\n<td><\/td>\n<td>5<\/td>\n<td>\u5e76\u53d1\u7ec4\u6570<\/td>\n<\/tr>\n<tr>\n<td>max-file-group-size-bytes<\/td>\n<td><\/td>\n<td>107374182400&#xff08;100GB&#xff09;<\/td>\n<td>\u5355\u7ec4\u4e0a\u9650<\/td>\n<\/tr>\n<tr>\n<td>use-starting-sequence-number<\/td>\n<td><\/td>\n<td>true<\/td>\n<td>\u5e8f\u5217\u53f7\u8bed\u4e49&#xff0c;\u4e0e MoR \u5220\u9664\u6587\u4ef6\u76f8\u5173<\/td>\n<\/tr>\n<tr>\n<td>partial-progress.enabled \/ max-commits<\/td>\n<td><\/td>\n<td>false \/ 10<\/td>\n<td>\u5206\u6279\u63d0\u4ea4<\/td>\n<\/tr>\n<tr>\n<td>output-spec-id<\/td>\n<td><\/td>\n<td>\u5f53\u524d spec<\/td>\n<td>\u8de8 partition evolution \u65f6\u5fc5\u987b\u663e\u5f0f\u6307\u5b9a<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u8fd9\u5f20\u8868\u91cc\u6700\u5bb9\u6613\u5199\u9519\u7684\u662f\u53c2\u6570\u540d\u3002 \u65e7\u7248\u6559\u7a0b\u91cc\u5e38\u89c1\u7684 sort_by \/ zorder_by \/ distribute_by \/ sort_buffer_size&#xff0c;\u5728\u672c\u6b21 1.12.0 \u7684 rewrite_data_files \u6587\u6863\u91cc\u51fa\u73b0\u6b21\u6570\u4e3a 0&#xff08;sort_by \u53ea\u5b58\u5728\u4e8e rewrite_manifests&#xff0c;\u8bed\u4e49\u662f&#034;\u6309\u5206\u533a\u53d8\u6362\u540d cluster manifest&#034;&#xff09;\u3002\u7167\u65e7\u7248\u53c2\u6570\u540d\u5199&#xff0c;\u4f1a\u76f4\u63a5\u62a5\u627e\u4e0d\u5230\u53c2\u6570\u2014\u2014\u8fd9\u4e0d\u662f\u517c\u5bb9\u6027\u95ee\u9898&#xff0c;\u662f\u7248\u672c\u57fa\u7ebf\u4e0d\u540c\u3002 \u7b2c 10 \u7ae0\u7684\u7ef4\u62a4\u4f5c\u4e1a\u4e00\u5f8b\u6309\u8fd9\u5f20\u8868\u5199\u3002<\/p>\n<p>where \u90a3\u53e5 \u201call files that may contain data matching\u201d \u4e5f\u8981\u5355\u72ec\u8bb0\u4f4f&#xff1a;\u5b83\u662f\u5019\u9009\u6587\u4ef6\u7b5b\u9009&#xff0c;\u4e0d\u662f\u884c\u8fc7\u6ee4\u3002rewrite \u4e0d\u4f1a\u51cf\u5c11\u4efb\u4f55\u884c&#xff0c;\u8c01\u8981\u662f\u4ee5\u4e3a where \u80fd&#034;\u6e05\u7406\u65e7\u6570\u636e&#034;&#xff0c;\u5c31\u4f1a\u5728\u8dd1\u5b8c\u4e4b\u540e\u53d1\u73b0\u6570\u636e\u8fd8\u5728\u2014\u2014\u800c\u771f\u6b63\u7684\u6e05\u7406\u662f DELETE &#043; expire&#xff08;\u7b2c 10 \u7ae0&#xff09;\u3002<\/p>\n<h4>5.4.4 \u4e0e ClickHouse \u7a00\u758f\u7d22\u5f15\u5bf9\u7167&#xff1a;\u88c1\u526a\u7c92\u5ea6\u5dee\u591a\u5c11<\/h4>\n<p>\u524d\u4f5c ck \u7b2c 3\/4 \u7ae0\u53d6\u8bc1\u8fc7 ClickHouse \u7684\u673a\u5236&#xff1a;ORDER BY \u6392\u5e8f\u952e\u5373\u4e3b\u952e\u524d\u7f00&#xff0c;part \u5185\u4e25\u683c\u6709\u5e8f&#xff0c;mark \u6bcf index_granularity&#xff08;\u9ed8\u8ba4 8192&#xff09;\u884c\u4e00\u6761&#xff0c;\u4e8c\u5206\u5b9a\u4f4d granule\u3002\u5bf9\u7167\u6e56\u8868&#xff1a;<\/p>\n<p>\u8868 5-7 ClickHouse \u7a00\u758f\u7d22\u5f15 vs \u6e56\u8868\u6392\u5e8f\u88c1\u526a<\/p>\n<table>\n<tr>\u5bf9\u6bd4\u9879ClickHouse ORDER BY &#043; markSpark \u6e56\u8868\u6392\u5e8f &#043; min\/max \u7edf\u8ba1<\/tr>\n<tbody>\n<tr>\n<td>\u987a\u5e8f\u7684\u627f\u8bfa\u5f3a\u5ea6<\/td>\n<td>\u5f3a&#xff1a;part \u4e0d\u53ef\u53d8&#xff0c;merge \u540e\u4ecd\u4e25\u683c\u6709\u5e8f<\/td>\n<td>\u5f31&#xff1a;\u53ea\u5728\u5199\u5165\/rewrite \u90a3\u4e00\u523b\u5f62\u6210&#xff0c;\u8ffd\u52a0\u5373\u7a00\u91ca<\/td>\n<\/tr>\n<tr>\n<td>\u5b9a\u4f4d\u65b9\u5f0f<\/td>\n<td>mark \u6587\u4ef6\u4e8c\u5206&#xff08;\u7cbe\u786e\u884c\u533a\u95f4&#xff09;<\/td>\n<td>\u7edf\u8ba1\u8fb9\u754c\u6bd4\u8f83&#xff08;\u53ea\u80fd\u5224&#034;\u53ef\u80fd\/\u4e0d\u53ef\u80fd&#034;&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u6700\u5c0f\u88c1\u526a\u5355\u4f4d<\/td>\n<td>1 granule \u2248 8192 \u884c&#xff08;\u9ed8\u8ba4&#xff09;<\/td>\n<td>1 row-group&#xff08;\u9ed8\u8ba4 128MB&#xff09;\u6216 1 \u6587\u4ef6&#xff08;\u9ed8\u8ba4 512MB&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u60f3\u8ba9\u7c92\u5ea6\u53d8\u7ec6<\/td>\n<td>\u8c03\u5c0f index_granularity&#xff0c;\u4ee3\u4ef7\u5c0f<\/td>\n<td>\u53ea\u80fd\u628a\u6587\u4ef6\u4e0e row-group \u5207\u5c0f&#xff0c;\u4ee3\u4ef7\u662f\u6587\u4ef6\u6570\u4e0e planning<\/td>\n<\/tr>\n<tr>\n<td>\u5931\u6548\u7684\u8868\u73b0<\/td>\n<td>\u8df3\u8fc7\u6392\u5e8f\u952e\u7b2c\u4e00\u5217\u3001\u51fd\u6570\u5305\u952e\u5217<\/td>\n<td>\u7edf\u8ba1\u8fb9\u754c\u8fc7\u5bbd&#xff08;\u524d\u7f00\u622a\u65ad\u3001\u7ef4\u5ea6\u4e0d\u76f8\u5173\u3001\u8ffd\u52a0\u7a00\u91ca&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u89c2\u6d4b\u624b\u6bb5<\/td>\n<td>EXPLAIN indexes &#061; 1 \u7684 Parts\/Granules A\/B<\/td>\n<td>files read\u3001scanned rows\u3001\u5143\u6570\u636e\u8868 files \u7684 bounds<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u4e00\u53e5\u8bdd\u6982\u62ec\u8fd9\u4e2a\u5bf9\u7167&#xff08;\u7b14\u8005\u5f52\u7eb3&#xff0c;\u975e\u5b98\u65b9\u63aa\u8f9e&#xff09;&#xff1a;<\/p>\n<p>ClickHouse \u7684\u6392\u5e8f&#034;\u5c31\u662f&#034;\u7d22\u5f15&#xff1b;\u6e56\u8868\u7684\u6392\u5e8f&#034;\u5582\u7ed9&#034;\u7edf\u8ba1\u4fe1\u606f\u3002\u524d\u8005\u7701\u7684\u662f\u884c\u533a\u95f4&#xff0c;\u540e\u8005\u7701\u7684\u662f\u6587\u4ef6\u4e0e\u5757\u3002 \u6240\u4ee5\u540c\u4e00\u53e5 sort(city_id, vin)&#xff0c;\u5728 CK \u4e0a\u4f60\u80fd\u6307\u671b 8192 \u884c\u7684\u7cbe\u5ea6&#xff0c;\u5728\u6e56\u8868\u4e0a\u4f60\u7684\u7cbe\u5ea6\u5355\u4f4d\u662f&#034;\u4e00\u4e2a\u51e0\u767e MB \u7684\u6587\u4ef6\u91cc\u6709\u51e0\u6bb5\u7a84\u8fb9\u754c&#034;\u2014\u2014\u9664\u975e\u4f60\u628a\u6587\u4ef6\u5207\u5c0f\u5230\u51e0\u5341 MB&#xff0c;\u800c\u90a3\u5c31\u7b49\u4e8e\u4eb2\u624b\u5236\u9020\u5c0f\u6587\u4ef6\u95ee\u9898\u3002<\/p>\n<p>\u8fd9\u6761\u5bf9\u7167\u7684\u76f4\u63a5\u5de5\u7a0b\u540e\u679c&#xff0c;\u662f\u672c\u7ae0\u7ed9\u6e56\u8868\u5b9a\u7684\u4e00\u4e2a\u52a1\u5b9e\u8d77\u70b9&#xff1a;<\/p>\n<p>   \u6e56\u8868\u4e0d\u8981\u8ffd\u6c42 granule \u7ea7\u7ec6\u7c92\u5ea6\u3002<br \/>\n   \u76ee\u6807 &#061; \u6bcf\u4e2a\u6587\u4ef6\/\u5757\u5185\u7684 city\u3001\u65e5\u671f\u9ad8\u5ea6\u540c\u8d28&#xff0c;\u4e14\u6587\u4ef6\u603b\u6570\u53ef\u63a7\u3002<br \/>\n   \u5e38\u89c1\u843d\u70b9&#xff1a;\u6587\u4ef6 32~128MB\u3001row-group \u7528\u9ed8\u8ba4\u6216\u6298\u534a\u3001\u5206\u533a\u6309\u5929\u3001<br \/>\n             \u6392\u5e8f\u5217\u4e0d\u8d85\u8fc7 3~4 \u4e2a&#xff08;\u8d85\u51fa\u5c31\u56de\u5230\u5b98\u65b9 auto \u89c4\u5219&#xff1a;2~4 \u7528 zorder&#xff0c;\u22655 \u7528 hilbert&#xff09;<br \/>\n   \u53cd\u4f8b&#xff1a;\u4e3a\u4e86\u8ba9&#034;\u67e5\u4e00\u53f0\u8f66&#034;\u66f4\u5feb&#xff0c;\u628a target \u8bbe\u6210 2MB \u21d2 \u6587\u4ef6\u6570 \u00d7250&#xff0c;<br \/>\n         manifest \u4e0e planning \u5148\u70b8&#xff0c;\u7b2c 10 \u7ae0\u6765\u6536\u62fe\u4f60<\/p>\n<hr \/>\n<h3>5.5 Bucket&#xff1a;\u4e09\u4e2a\u5f15\u64ce\u540c\u4e00\u4e2a\u8bcd&#xff0c;\u4e09\u79cd\u8bed\u4e49<\/h3>\n<p>&#034;\u5206\u6876&#034;\u662f\u5168\u7cfb\u5217\u6700\u5bb9\u6613\u8ba9\u4eba\u628a\u4e0d\u540c\u4e1c\u897f\u5f53\u6210\u540c\u4e00\u4ef6\u4e8b\u7684\u8bcd\u3002\u7b2c 4 \u7ae0\u8bb2\u8fc7 Spark \u7684 join \u7b56\u7565\u9009\u62e9&#xff0c;\u524d\u4f5c\u5728 Doris \u4fa7\u5b9a\u8fc7 Colocate Join \u7684\u786c\u7ea6\u675f&#xff08;\u540c\u5206\u6876\u5217\u3001\u540c\u6876\u6570\u3001\u540c colocate group&#xff09;&#xff0c;\u7b2c 2 \u7ae0\u53c8\u7ed9 Paimon \u8868\u5199\u4e86 &#039;bucket&#039; &#061; &#039;4&#039;\u3002\u4e09\u4e2a&#034;\u6876&#034;\u7684\u76ee\u7684\u5b8c\u5168\u4e0d\u540c&#xff0c;\u628a\u5b83\u4eec\u5f53\u540c\u4e00\u4e2a\u4e1c\u897f&#xff0c;\u5c31\u4f1a\u5728\u9519\u8bef\u7684\u573a\u666f\u671f\u5f85\u9519\u8bef\u7684\u6536\u76ca\u3002<\/p>\n<h4>5.5.1 \u4e09\u5957 bucket \u7684\u8bed\u4e49\u5bf9\u7167<\/h4>\n<p>\u8868 5-8 \u4e09\u79cd bucket \u5bf9\u7167<\/p>\n<table>\n<tr>\u7ef4\u5ea6Hive \u98ce\u683c bucketIceberg bucket(n) transformPaimon bucket<\/tr>\n<tbody>\n<tr>\n<td>\u5b83\u672c\u8d28\u662f\u4ec0\u4e48<\/td>\n<td>\u8868\u7684\u4e00\u4e2a\u7269\u7406\u5c5e\u6027&#xff1a;hash(\u6876\u5217) % n \u51b3\u5b9a\u884c\u843d\u5230\u54ea\u4e2a\u6876\u6587\u4ef6<\/td>\n<td>\u4e00\u79cd\u5206\u533a\u53d8\u6362&#xff1a;\u628a\u9ad8\u57fa\u6570\u5217\u79bb\u6563\u6210\u6709\u9650\u4e2a&#034;\u5206\u533a\u503c&#034;<\/td>\n<td>\u4e3b\u952e\u8868\u91cc&#034;\u4e3b\u952e \u2192 \u6587\u4ef6&#034;\u7684\u5b9a\u4f4d\u5355\u4f4d<\/td>\n<\/tr>\n<tr>\n<td>\u8bbe\u8ba1\u76ee\u7684<\/td>\n<td>\u2460 join\/\u805a\u5408\u514d shuffle \u2461 \u6876\u5185\u9884\u6392\u5e8f\u514d sort \u2462 \u62bd\u6837<\/td>\n<td>\u7ed9\u9ad8\u57fa\u6570\u5217\u505a\u88c1\u526a&#xff08;\u65f6\u95f4\u5217\u4e4b\u5916\u7684\u7b2c\u4e8c\u7ef4\u5ea6&#xff09;<\/td>\n<td>\u63a7\u5236\u5e76\u53d1\u5199\u5165\u4e0e LSM \u5c42\u7ea7\u89c4\u6a21&#xff1b;\u4fdd\u8bc1\u540c\u952e\u6570\u636e\u540c\u6876<\/td>\n<\/tr>\n<tr>\n<td>\u5199\u4fa7\u5f62\u6001<\/td>\n<td>CLUSTERED BY (vin) SORTED BY (vin, reported_at) INTO 8 BUCKETS<\/td>\n<td>PARTITIONED BY (bucket(16, vin))<\/td>\n<td>bucket &gt; 0 \u56fa\u5b9a\u6876&#xff1b;\u5b98\u65b9\u9ed8\u8ba4 -1 \u52a8\u6001\u6876&#xff1b;-2 \u4e3a postpone \u6a21\u5f0f\u3002bucket-key \u4e0d\u7ed9\u65f6&#034;\u7528\u4e3b\u952e&#034;&#xff08;Paimon \u5b98\u65b9&#xff1a;\u201cFor primary-key tables, bucket-key defaults to the primary key excluding partition columns.\u201d&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u5b9a\u4f4d\u7b97\u6cd5<\/td>\n<td>hash(\u6876\u5217) % n<\/td>\n<td>bucket[N]&#xff1a;\u201cHash of value, mod N\u201d&#xff08;Iceberg spec&#xff09;<\/td>\n<td>\u5b98\u65b9\u9010\u5b57&#xff1a;\u201cPaimon calculates the bucket using Math.abs(key_hashcode % numBuckets)\u201d<\/td>\n<\/tr>\n<tr>\n<td>\u8bfb\u4fa7\u6536\u76ca\u6761\u4ef6<\/td>\n<td>\u4e24\u4fa7\u6876\u5217\u8bed\u4e49\u76f8\u540c &#043; \u6876\u6570\u76f8\u540c \u21d2 \u53ef\u7701 shuffle&#xff1b;\u5355\u8868\u6309\u6876\u5217\u7b49\u503c\u4e5f\u53ef\u53ea\u8bfb\u90e8\u5206\u6876<\/td>\n<td>\u5bf9 vin \u5199\u7b49\u503c\/IN \u21d2 \u53ea\u626b\u5bf9\u5e94\u6876\u5206\u533a&#xff1b;\u8303\u56f4\u6761\u4ef6 \u21d2 \u5168\u6876\u626b<\/td>\n<td>\u5bf9 bucket-key\/\u4e3b\u952e\u5199\u7b49\u503c \u21d2 \u5b9a\u4f4d\u5230 1 \u4e2a\u6876&#xff1b;\u8303\u56f4 \u21d2 \u5168\u6876\u626b<\/td>\n<\/tr>\n<tr>\n<td>\u6876\u6570\u80fd\u4e0d\u80fd\u6539<\/td>\n<td>\u6539\u6876\u6570\u9700\u91cd\u5199\u6574\u8868<\/td>\n<td>\u5c5e\u4e8e partition spec \u53d8\u66f4&#xff0c;\u8d70 partition evolution&#xff0c;\u8001\u6570\u636e\u4ecd\u6309\u65e7 spec&#xff08;\u5b98\u65b9&#xff1a;\u201cPartition evolution is a metadata operation and does not eagerly rewrite files.\u201d&#xff09;<\/td>\n<td>\u5b98\u65b9\u4e09\u6a21\u5f0f\u7684\u89c4\u5212\u7ea6\u675f\u9010\u5b57&#xff1a;\u56fa\u5b9a\u6876 \u201cPredictable layout; resizing requires an explicit rescale operation\u201d&#xff1b;\u52a8\u6001\u6876 \u201cIndex memory and single-writer constraints\u201d&#xff1b;postpone \u201cBatch\/compaction workflow and per-partition sizing\u201d<\/td>\n<\/tr>\n<tr>\n<td>\u6700\u5e38\u89c1\u7684\u8bef\u7528<\/td>\n<td>\u6876\u6570\u4e0e shuffle.partitions \u65e0\u5173\u7cfb\u5374\u88ab\u5f53\u6210\u5e76\u884c\u5ea6\u8bbe\u7f6e<\/td>\n<td>\u6876\u6570\u53d6\u592a\u5c0f&#xff08;\u5982 4&#xff09;&#xff0c;&#034;\u6876&#034;\u53d8\u6210\u7c97\u5206\u533a&#xff0c;\u88c1\u526a\u7c92\u5ea6\u53cd\u800c\u53d8\u5dee<\/td>\n<td>\u628a\u6876\u6570\u6309&#034;\u5168\u8868&#034;\u800c\u4e0d\u662f&#034;\u6309\u5206\u533a&#034;\u7b97&#xff0c;\u5bfc\u81f4\u5355\u5206\u533a\u6587\u4ef6\u6570\u5931\u63a7<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u4e09\u6761\u5fc5\u987b\u8bf4\u900f\u7684\u5dee\u5f02&#xff1a;<\/p>\n<p>\u2460 Iceberg \u7684 bucket \u4e0d\u662f&#034;\u6876&#034;&#xff0c;\u662f&#034;\u5206\u533a&#034;\u3002 bucket(16, vin) \u51fa\u73b0\u5728 partition spec \u91cc&#xff0c;\u4ea7\u51fa\u7684 vin_bucket&#061;7\/ \u4e0e report_date_day&#061;2026-10-15\/ \u662f\u540c\u4e00\u5c42\u7ea7\u7684\u4e1c\u897f\u3002\u6240\u4ee5\u5b83\u7684\u4ee3\u4ef7\u4e5f\u6309\u5206\u533a\u7b97&#xff1a;16 \u4e2a\u6876 \u00d7 30 \u5929 &#061; 480 \u4e2a\u5206\u533a\u76ee\u5f55\u3002\u8fd9\u5c31\u662f\u4e3a\u4ec0\u4e48 Iceberg \u4fa7\u9009\u6876\u6570\u8981\u62ff&#034;\u51e0\u5343\u91cf\u7ea7\u5206\u533a&#034;\u8fd9\u628a\u5c3a\u5b50\u91cf&#xff0c;\u800c\u4e0d\u662f\u62ff&#034;\u5e76\u884c\u5ea6&#034;\u8fd9\u628a\u3002<\/p>\n<p>\u2461 Paimon \u7684\u6876\u662f&#034;\u6b63\u786e\u6027\u7ea6\u675f&#034;&#xff0c;\u4e0d\u53ea\u662f\u6027\u80fd\u9009\u9879\u3002 \u5b98\u65b9\u90a3\u53e5\u8bdd\u628a\u56e0\u679c\u8bb2\u5b8c\u4e86&#xff1a;\u201cFor primary-key tables, bucket-key defaults to the primary key excluding partition columns.\u201d \u2014\u2014 \u9ed8\u8ba4\u6876\u952e\u5c31\u662f&#034;\u4e3b\u952e\u53bb\u6389\u5206\u533a\u5217&#034;\u3002\u628a\u5b83\u548c\u7b2c 2 \u7ae0\u90a3\u6761\u786c\u7ea6\u675f&#xff08;Paimon \u4e3b\u952e\u5fc5\u987b\u5305\u542b\u5168\u90e8\u5206\u533a\u5b57\u6bb5&#xff09;\u653e\u5728\u4e00\u8d77\u8bfb&#xff0c;\u7ed3\u8bba\u5f88\u6e05\u695a&#xff1a;\u5206\u533a\u5b57\u6bb5\u8d1f\u8d23\u628a\u6570\u636e\u5207\u5230\u76ee\u5f55&#xff0c;\u5269\u4e0b\u7684\u4e3b\u952e\u90e8\u5206\u8d1f\u8d23\u5b9a\u6876&#xff0c;\u4e24\u8005\u62fc\u8d77\u6765\u624d\u662f\u5b8c\u6574\u4e3b\u952e\u3002\u8c01\u624b\u52a8\u6307\u5b9a\u4e86\u4e00\u4e2a\u4e0e\u4e3b\u952e\u4e0d\u4e00\u81f4\u7684 bucket-key&#xff0c;\u8c01\u5c31\u5728\u5236\u9020&#034;\u540c\u952e\u4e0d\u540c\u6876&#034;\u7684\u98ce\u9669\u3002<\/p>\n<p>\u8fd8\u6709\u4e00\u4e2a\u8bfb\u8005\u4e00\u5b9a\u4f1a\u649e\u4e0a\u7684\u5dee\u5f02&#xff1a;Paimon \u5b98\u65b9\u9ed8\u8ba4\u662f\u52a8\u6001\u6876&#xff08;bucket &#061; -1&#xff09;&#xff0c;\u800c\u7b2c 2 \u7ae0\u7684 ODS \u8868\u663e\u5f0f\u5199\u4e86 &#039;bucket&#039; &#061; &#039;4&#039;&#xff08;\u56fa\u5b9a\u6876&#xff09;\u3002 \u8fd9\u662f\u523b\u610f\u5b89\u6392\u7684\u2014\u2014\u5b9e\u9a8c\u8981\u7684\u662f\u53ef\u9884\u671f\u7684\u76ee\u5f55\u5f62\u6001\u3002\u751f\u4ea7\u4e0a\u9009\u56fa\u5b9a\u6876\u610f\u5473\u7740\u63a5\u53d7\u5b98\u65b9\u90a3\u53e5 \u201cresizing requires an explicit rescale operation\u201d&#xff1b;\u9009\u52a8\u6001\u6876\u610f\u5473\u7740\u63a5\u53d7 \u201cIndex memory and single-writer constraints\u201d&#xff08;\u7d22\u5f15\u5360\u5185\u5b58\u3001\u5355\u5199\u8005\u7ea6\u675f&#xff09;\u3002\u8fd9\u4e24\u4e2a\u4ee3\u4ef7\u5fc5\u987b\u5728\u5efa\u8868\u8bc4\u5ba1\u65f6\u5199\u4e0b\u6765&#xff0c;\u4e0d\u80fd\u7b49\u5230\u6539\u6876\u6570\u90a3\u5929\u624d\u53d1\u73b0\u3002<\/p>\n<p>\u2462 Hive bucket \u662f\u552f\u4e00\u80fd\u5e26\u6765&#034;\u514d shuffle&#034;\u7684&#xff0c;\u4f46\u5b83\u6709\u524d\u7f6e\u95e8\u69db\u3002 Spark \u4fa7\u8981\u8ba4\u51fa\u8fd9\u4e2a\u6027\u8d28&#xff0c;\u9700\u8981&#xff1a;\u5206\u6876\u4fe1\u606f\u5728 catalog \u91cc\u53ef\u89c1&#xff0c;\u4e14 spark.sql.sources.bucketing.enabled \u5904\u4e8e\u5f00\u542f&#xff08;\u5b98\u65b9\u9ed8\u8ba4 true&#xff0c;2.0.0 \u8d77&#xff1b;\u539f\u6587\u63aa\u8f9e\u5f88\u6709\u753b\u9762\u611f\u2014\u2014\u201cWhen false, we will treat bucketed table as normal table\u201d&#xff09;&#xff0c;\u53e6\u6709\u4e0a\u9650 spark.sql.sources.bucketing.maxBuckets &#061; 100000&#xff08;2.4.0 \u8d77&#xff09;\u3002\u5c11\u4efb\u4f55\u4e00\u6761&#xff0c;\u6536\u76ca\u9759\u9ed8\u5f52\u96f6\u2014\u2014\u8ba1\u5212\u91cc Exchange \u4e00\u4e2a\u4e0d\u5c11&#xff0c;\u800c\u4e14\u6ca1\u6709\u4efb\u4f55\u8b66\u544a\u3002<\/p>\n<h4>5.5.2 \u514d shuffle \u8fd9\u4ef6\u4e8b&#xff1a;\u5148\u9a8c\u8bc1&#xff0c;\u518d\u671f\u5f85<\/h4>\n<p><span class=\"token comment\">&#8212; \u524d\u7f6e\u68c0\u67e5 1&#xff1a;\u5206\u6876\u4f18\u5316\u5f00\u5173\u5728\u4e0d\u5728&#xff08;\u5b98\u65b9\u9ed8\u8ba4 true&#xff09;<\/span><br \/>\n<span class=\"token keyword\">SET<\/span> spark<span class=\"token punctuation\">.<\/span><span class=\"token keyword\">sql<\/span><span class=\"token punctuation\">.<\/span>sources<span class=\"token punctuation\">.<\/span>bucketing<span class=\"token punctuation\">.<\/span>enabled<span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u8f93\u51fa\u5217&#xff1a;key(STRING), value(STRING)<\/span><br \/>\n<span class=\"token comment\">&#8212; \u82e5\u88ab\u67d0\u5904\u914d\u7f6e\u6210 false&#xff0c;\u5b98\u65b9\u884c\u4e3a\u662f&#034;\u628a\u5206\u6876\u8868\u5f53\u666e\u901a\u8868&#034;\u2014\u2014\u6536\u76ca\u76f4\u63a5\u5f52\u96f6\u4e14\u4e0d\u62a5\u9519<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u524d\u7f6e\u68c0\u67e5 2&#xff1a;\u6876\u4e0a\u9650<\/span><br \/>\n<span class=\"token keyword\">SET<\/span> spark<span class=\"token punctuation\">.<\/span><span class=\"token keyword\">sql<\/span><span class=\"token punctuation\">.<\/span>sources<span class=\"token punctuation\">.<\/span>bucketing<span class=\"token punctuation\">.<\/span>maxBuckets<span class=\"token punctuation\">;<\/span>    <span class=\"token comment\">&#8212; \u5b98\u65b9\u9ed8\u8ba4 100000<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u524d\u7f6e\u68c0\u67e5 3&#xff1a;\u5f53\u524d catalog \u80fd\u5426\u6301\u4e45\u5316\u5206\u6876\u5143\u6570\u636e<\/span><br \/>\n<span class=\"token keyword\">SET<\/span> spark<span class=\"token punctuation\">.<\/span><span class=\"token keyword\">sql<\/span><span class=\"token punctuation\">.<\/span>catalogImplementation<span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u8bf4\u660e&#xff1a;\u5b98\u65b9 CREATE TABLE \u7684\u4e24\u4e2a\u9875\u9762\u90fd\u63a5\u53d7 CLUSTERED BY \u5b50\u53e5\u2014\u2014<\/span><br \/>\n<span class=\"token comment\">&#8212;   Hive \u98ce\u683c&#xff1a;sql-ref-syntax-ddl-create-table-hiveformat.html<\/span><br \/>\n<span class=\"token comment\">&#8212;   DataSource \u98ce\u683c&#xff1a;sql-ref-syntax-ddl-create-table-datasource.html<\/span><br \/>\n<span class=\"token comment\">&#8212;   \u5b98\u65b9\u539f\u6587\u793a\u4f8b\u9010\u5b57\u4e3a&#xff1a;CREATE TABLE student (id INT, name STRING, age INT)<\/span><br \/>\n<span class=\"token comment\">&#8212;                       USING CSV PARTITIONED BY (age) CLUSTERED BY (Id) INTO 4 buckets;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u4e24\u79cd\u5199\u6cd5\u90fd\u80fd\u5efa&#xff0c;\u5dee\u522b\u5728&#034;\u6876\u4fe1\u606f\u5199\u8fdb\u54ea\u4efd\u5143\u6570\u636e\u3001\u80fd\u4e0d\u80fd\u88ab\u4e0b\u6b21\u4f1a\u8bdd\u8bfb\u56de\u6765&#034;\u3002<\/span><br \/>\n<span class=\"token comment\">&#8212; \u5efa\u5b8c\u7acb\u523b DESCRIBE EXTENDED \u770b\u4e00\u773c&#xff0c;\u6bd4\u80cc\u8bed\u6cd5\u53ef\u9760\u3002<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u6587\u4ef6&#xff1a;partition_design.sql \u7684\u7b2c 3 \u6bb5<\/span><br \/>\n<span class=\"token comment\">&#8212; \u5b98\u65b9\u4e24\u79cd DDL \u5f62\u6001\u90fd\u63a5\u53d7 CLUSTERED BY&#xff1b;\u751f\u4ea7\u4e0a\u63a8\u8350\u843d\u8fdb\u80fd\u6301\u4e45\u5316\u6876\u4fe1\u606f\u7684 catalog&#xff08;\u63a5 HMS \u7684 spark_catalog&#xff09;<\/span><br \/>\n<span class=\"token keyword\">CREATE<\/span> <span class=\"token keyword\">TABLE<\/span> <span class=\"token keyword\">IF<\/span> <span class=\"token operator\">NOT<\/span> <span class=\"token keyword\">EXISTS<\/span> spark_catalog<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>bkt_telemetry <span class=\"token punctuation\">(<\/span><br \/>\n    vin          STRING<span class=\"token punctuation\">,<\/span><br \/>\n    report_date  <span class=\"token keyword\">DATE<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    reported_at  <span class=\"token keyword\">TIMESTAMP<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    city_id      <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    speed        <span class=\"token keyword\">FLOAT<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    battery      <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    vehicle_status STRING<span class=\"token punctuation\">,<\/span><br \/>\n    seq          <span class=\"token keyword\">BIGINT<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">USING<\/span> parquet<br \/>\n<span class=\"token keyword\">CLUSTERED<\/span> <span class=\"token keyword\">BY<\/span> <span class=\"token punctuation\">(<\/span>vin<span class=\"token punctuation\">)<\/span>                <span class=\"token comment\">&#8212; \u6876\u952e&#xff1a;join \u952e&#xff0c;\u5fc5\u987b\u4e0e\u4e0b\u6e38\u67e5\u8be2\u7684\u7b49\u503c\u5217\u4e00\u81f4<\/span><br \/>\nSORTED <span class=\"token keyword\">BY<\/span> <span class=\"token punctuation\">(<\/span>vin<span class=\"token punctuation\">,<\/span> reported_at<span class=\"token punctuation\">)<\/span>      <span class=\"token comment\">&#8212; \u6876\u5185\u6392\u5e8f&#xff1a;\u8ba9&#034;\u6309\u8f66\u53d6\u65f6\u5e8f&#034;\u514d\u4e00\u6b21 sort<\/span><br \/>\n<span class=\"token keyword\">INTO<\/span> <span class=\"token number\">8<\/span> BUCKETS                      <span class=\"token comment\">&#8212; 8 &#061; 2 \u7684\u5e42&#xff0c;\u4fbf\u4e8e\u4e0e\u5e76\u884c\u5ea6\u5bf9\u9f50<\/span><br \/>\nPARTITIONED <span class=\"token keyword\">BY<\/span> <span class=\"token punctuation\">(<\/span>report_date<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u704c\u6570&#xff1a;\u76f4\u63a5\u6f14\u793a&#034;\u540c\u6876\u952e\u540c\u6876\u6570&#034;\u7684\u7b2c\u4e8c\u5f20\u8868&#xff08;\u805a\u5408\u7ed3\u679c\u8868&#xff0c;\u7528\u4e8e join&#xff09;<\/span><br \/>\n<span class=\"token keyword\">CREATE<\/span> <span class=\"token keyword\">TABLE<\/span> <span class=\"token keyword\">IF<\/span> <span class=\"token operator\">NOT<\/span> <span class=\"token keyword\">EXISTS<\/span> spark_catalog<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>bkt_vehicle_daily <span class=\"token punctuation\">(<\/span><br \/>\n    vin          STRING<span class=\"token punctuation\">,<\/span><br \/>\n    report_date  <span class=\"token keyword\">DATE<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    trip_cnt     <span class=\"token keyword\">BIGINT<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    avg_speed    <span class=\"token keyword\">DECIMAL<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">,<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">USING<\/span> parquet<br \/>\n<span class=\"token keyword\">CLUSTERED<\/span> <span class=\"token keyword\">BY<\/span> <span class=\"token punctuation\">(<\/span>vin<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">INTO<\/span> <span class=\"token number\">8<\/span> BUCKETS<br \/>\nPARTITIONED <span class=\"token keyword\">BY<\/span> <span class=\"token punctuation\">(<\/span>report_date<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff08;\u53ea\u8bb2\u5bb9\u6613\u8bef\u89e3\u7684\u4e24\u5904&#xff09;<\/p>\n<table>\n<tr>\u9879\u53d6\u503c\u4e3a\u4ec0\u4e48<\/tr>\n<tbody>\n<tr>\n<td>CLUSTERED BY (vin)<\/td>\n<td>\u5355\u5217&#xff0c;\u7b49\u4e8e join \u952e<\/td>\n<td>\u591a\u5217\u6876\u952e\u8981\u6c42 join \u6761\u4ef6\u8986\u76d6\u5168\u90e8\u6876\u5217\u624d\u5bf9\u9f50&#xff0c;\u8f66\u8054\u7f51\u573a\u666f\u91cc vin \u5355\u5217\u6700\u5b9e\u7528<\/td>\n<\/tr>\n<tr>\n<td>SORTED BY (vin, reported_at)<\/td>\n<td>\u6876\u5185\u4e8c\u7ea7\u6392\u5e8f<\/td>\n<td>\u514d shuffle \u4e4b\u5916\u518d\u514d\u4e00\u6b21 sort&#xff1b;\u4ee3\u4ef7\u662f\u5199\u5165\u7aef\u591a\u4ed8\u4e00\u6b21\u6392\u5e8f&#xff0c;\u5199\u653e\u5927\u6362\u8bfb\u52a0\u901f<\/td>\n<\/tr>\n<tr>\n<td>INTO 8 BUCKETS<\/td>\n<td>8<\/td>\n<td>\u6876\u6570\u4e00\u65e6\u5b9a\u4e0b\u4e0d\u53ef\u8f7b\u6613\u6539\u3002\u9009\u62e9\u4f9d\u636e\u89c1\u4e0b<\/td>\n<\/tr>\n<tr>\n<td>PARTITIONED BY (report_date)<\/td>\n<td>\u4e0e\u5206\u6876\u540c\u65f6\u5b58\u5728<\/td>\n<td>\u5206\u533a\u7ba1&#034;\u54ea\u4e9b\u5929\u7684\u6587\u4ef6&#034;&#xff0c;\u6876\u7ba1&#034;\u6bcf\u4e2a\u5206\u533a\u5185\u7684\u6587\u4ef6\u5f52\u5c5e&#034;&#xff0c;\u4e24\u5c42\u72ec\u7acb&#xff08;\u56fe 5-1&#xff09;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u6876\u6570\u600e\u4e48\u5b9a&#xff08;\u53e3\u5f84\u8981\u5206\u6e05&#xff09;<\/p>\n<p>   Hive \/ Paimon \u4fa7&#xff1a;\u6309&#034;\u5355\u6876\u6570\u636e\u91cf&#034;\u53cd\u63a8<br \/>\n        \u6876\u6570 \u2248 \u5355\u4e2a\u5206\u533a\u7684\u5b57\u8282\u91cf \u00f7 \u76ee\u6807\u5355\u6876\u5927\u5c0f<br \/>\n        Paimon \u5b98\u65b9\u53e3\u5f84&#xff08;\u9010\u5b57&#xff09;&#xff1a;&#034;For MOR workloads, 200 MB\u20131 GB per bucket is<br \/>\n        an initial sizing guide, not a limit. Measure skew, throughput, and read<br \/>\n        plans to select the actual count.&#034;<br \/>\n        \u21d2 \u4e00\u4e2a\u5206\u533a 4GB\u3001\u76ee\u6807 512MB \u21d2 8 \u6876<br \/>\n        \u21d2 \u6ce8\u610f\u5b98\u65b9\u7d27\u63a5\u7740\u7684\u90a3\u53e5&#xff1a;&#034;A batch scan can split one bucket into<br \/>\n          independent key ranges, so bucket count is not a hard read-parallelism cap.&#034;<br \/>\n          \u8fd9\u53e5\u76f4\u63a5\u5426\u6389\u4e86&#034;\u6876\u6570 &#061; \u8bfb\u5e76\u884c\u5ea6&#034;\u7684\u5e38\u89c1\u8bef\u89e3&#xff08;\u89c1 5.5.4&#xff09;<\/p>\n<p>   Iceberg \u4fa7&#xff1a;\u6309&#034;\u603b\u5206\u533a\u6570&#034;\u53cd\u63a8<br \/>\n        \u603b\u5206\u533a\u6570 \u2248 \u65f6\u95f4\u5206\u533a\u6570 \u00d7 bucket(n) \u7684 n<br \/>\n        \u53d7&#034;\u51e0\u5343\u91cf\u7ea7\u5206\u533a&#034;\u7ea6\u675f \u21d2 n \u7684\u4e0a\u754c\u7531\u65f6\u95f4\u5206\u533a\u6570\u51b3\u5b9a<br \/>\n        \u21d2 30 \u5929 \u00d7 n \u2264 \u51e0\u5343 \u21d2 n \u53ef\u4ee5\u5f88\u5927&#xff1b;\u4f46 1000 \u5929 \u00d7 n \u5c31\u5fc5\u987b\u628a n \u538b\u5230\u4e2a\u4f4d\u6570<\/p>\n<p>&#034;\u51e0\u5343\u91cf\u7ea7\u5206\u533a&#034;\u8fd9\u6761\u5c3a\u5b50\u8981\u8bf4\u6e05\u6765\u6e90&#xff1a;\u5b83\u662f\u524d\u4f5c Iceberg \u7cfb\u5217\u5b9a\u4e0b\u7684\u5de5\u7a0b\u53e3\u5f84&#xff0c;\u672c\u6b21\u5728 Iceberg 1.12.0 \u7684 partitioning \u6587\u6863\u5168\u6587\u91cc\u6ca1\u6709\u68c0\u7d22\u5230\u5bf9\u5e94\u63aa\u8f9e&#xff08;\u5b98\u65b9\u9875\u9762\u53ea\u8bb2 transform \u5217\u8868\u3001\u5206\u533a\u6f14\u8fdb\u4e0e&#034;plan \u53ea\u8bfb\u5339\u914d\u5206\u533a&#034;\u7684\u6536\u76ca&#xff09;\u3002\u6240\u4ee5\u5b83\u7684\u6027\u8d28\u662f\u5de5\u7a0b\u7ecf\u9a8c&#xff0c;\u4e0d\u662f\u5b98\u65b9\u89c4\u8303\u2014\u2014\u7528\u4e8e\u8bc4\u5ba1\u63d0\u95ee\u5f88\u6709\u7528&#xff0c;\u7528\u4e8e&#034;\u5f15\u7528\u5b98\u65b9&#034;\u4e0d\u884c\u3002<\/p>\n<p>\u8f93\u51fa\u8868\u7ed3\u6784&#xff1a;bkt_telemetry \u4e3a\u5206\u6876 &#043; \u5206\u533a\u8868&#xff0c;\u7269\u7406\u4e0a report_date&#061;2026-10-15\/000000_0.parquet &#8230;  \u6bcf\u5206\u533a 8 \u4e2a\u6876\u6587\u4ef6&#xff08;Spark \u5199 Hive \u98ce\u683c\u5206\u6876\u8868\u65f6&#xff0c;\u6587\u4ef6\u540d\u91cc\u7684\u7f16\u53f7\u627f\u8f7d\u6876\u53f7&#xff1b;\u5177\u4f53\u5f62\u6001\u968f\u7248\u672c\u800c\u5f02&#xff0c;\u9a8c\u6536\u4e0d\u770b\u6587\u4ef6\u540d&#xff09;\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6<\/p>\n<p><span class=\"token comment\">&#8212; \u5173\u952e\u9a8c\u6536&#xff1a;join \u7684 Exchange \u6709\u6ca1\u6709\u5c11<\/span><br \/>\n<span class=\"token keyword\">EXPLAIN<\/span> FORMATTED<br \/>\n<span class=\"token keyword\">SELECT<\/span> t<span class=\"token punctuation\">.<\/span>vin<span class=\"token punctuation\">,<\/span> t<span class=\"token punctuation\">.<\/span>report_date<span class=\"token punctuation\">,<\/span> t<span class=\"token punctuation\">.<\/span>speed<span class=\"token punctuation\">,<\/span> d<span class=\"token punctuation\">.<\/span>avg_speed<br \/>\n<span class=\"token keyword\">FROM<\/span> spark_catalog<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>bkt_telemetry t<br \/>\n<span class=\"token keyword\">JOIN<\/span> spark_catalog<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>bkt_vehicle_daily d<br \/>\n  <span class=\"token keyword\">ON<\/span> t<span class=\"token punctuation\">.<\/span>vin <span class=\"token operator\">&#061;<\/span> d<span class=\"token punctuation\">.<\/span>vin <span class=\"token operator\">AND<\/span> t<span class=\"token punctuation\">.<\/span>report_date <span class=\"token operator\">&#061;<\/span> d<span class=\"token punctuation\">.<\/span>report_date<span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u5224\u5b9a\u8868&#xff1a;<\/span><br \/>\n<span class=\"token comment\">&#8212; | \u89c2\u5bdf | \u7ed3\u8bba |<\/span><br \/>\n<span class=\"token comment\">&#8212; | join \u4e24\u4fa7\u5747\u65e0 Exchange hashpartitioning(vin, N) | \u5206\u6876\u5bf9\u9f50\u751f\u6548&#xff0c;\u514d shuffle |<\/span><br \/>\n<span class=\"token comment\">&#8212; | \u4e24\u4fa7\u5404\u591a\u4e00\u4e2a Exchange hashpartitioning(vin, 200) | \u6536\u76ca\u5f52\u96f6&#xff1a;\u6876\u6570\u4e0d\u540c \/ \u5f00\u5173\u672a\u5f00 \/ \u6876\u4fe1\u606f\u672a\u88ab\u8bc6\u522b |<\/span><\/p>\n<p>\u4e3a\u4ec0\u4e48&#034;\u5f52\u96f6&#034;\u8981\u5199\u6210\u4e00\u6761\u7eaa\u5f8b&#xff1a;\u514d shuffle \u5931\u6548\u65f6 Spark \u4e0d\u62a5\u9519\u3001\u4e0d\u6253\u8b66\u544a&#xff0c;\u8ba1\u5212\u9000\u5316\u5f97\u65e0\u58f0\u65e0\u606f\u3002\u552f\u4e00\u7684\u53d1\u73b0\u624b\u6bb5\u662f\u5bf9\u6bd4 Exchange \u6570\u91cf\u2014\u2014\u6240\u4ee5\u8fd9\u6761 SQL \u5e94\u8be5\u5728\u6539\u6876\u6570\u3001\u5347\u7ea7 Spark\u3001\u6362 catalog \u4e4b\u540e\u5404\u8dd1\u4e00\u6b21&#xff0c;\u628a\u7ed3\u679c\u5b58\u6863\u3002<\/p>\n<h4>5.5.3 \u4e0e Doris Colocate Join \u7684\u4e0d\u7b49\u4ef7\u6027&#xff08;\u56de\u6263\u524d\u4f5c&#xff09;<\/h4>\n<p>\u524d\u4f5c\u786c\u7ea6\u675f&#xff1a;Colocate Join \u8981\u6c42\u540c\u5206\u6876\u5217\u3001\u540c\u6876\u6570&#xff0c;\u5e76\u4e14\u8981\u914d\u7f6e Colocate Group\u3002Spark \u4fa7\u7684\u5206\u6876 join \u8868\u9762\u50cf&#xff0c;\u5b9e\u8d28\u5dee\u4e09\u70b9&#xff1a;<\/p>\n<table>\n<tr>\u5bf9\u6bd4\u9879Doris Colocate JoinSpark bucketed join<\/tr>\n<tbody>\n<tr>\n<td>&#034;\u540c\u6876&#034;\u4fdd\u8bc1\u7684\u662f\u4ec0\u4e48<\/td>\n<td>\u6570\u636e\u5728\u540c\u4e00 BE \u8282\u70b9\u4e0a\u7269\u7406\u5171\u7f6e&#xff08;tablet \u5206\u5e03\u4e00\u81f4&#xff09;<\/td>\n<td>\u4e24\u4fa7\u7684 hash \u5206\u533a\u8868\u8fbe\u5f0f\u4e00\u81f4&#xff0c;\u4ece\u800c\u7701\u6389\u4e00\u6b21\u91cd\u5206\u5e03<\/td>\n<\/tr>\n<tr>\n<td>\u7701\u6389\u7684\u4ee3\u4ef7<\/td>\n<td>\u8de8\u8282\u70b9\u7f51\u7edc\u4f20\u8f93 &#043; \u91cd\u5206\u5e03<\/td>\n<td>shuffle write &#043; shuffle read&#xff08;\u4ecd\u662f\u7f51\u7edc IO&#xff0c;\u53ea\u662f\u4e0d\u518d\u91cd\u6392&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u4e0d\u6ee1\u8db3\u65f6\u7684\u9000\u8def<\/td>\n<td>\u4e0d\u80fd colocate \u5c31\u8d70 Broadcast\/SHJ&#xff0c;\u9700\u8981\u663e\u5f0f\u5efa group<\/td>\n<td>\u9759\u9ed8\u9000\u56de\u666e\u901a SMJ&#xff0c;\u6ca1\u6709\u4efb\u4f55\u63d0\u793a<\/td>\n<\/tr>\n<tr>\n<td>\u6876\u7684\u8f7d\u4f53<\/td>\n<td>\u5efa\u8868\u65f6 DISTRIBUTED BY HASH(k) BUCKETS n &#043; PROPERTIES(&#034;colocate_with&#034;&#061;\u2026)<\/td>\n<td>\u5efa\u8868\u65f6 CLUSTERED BY (k) INTO n BUCKETS&#xff08;Hive \u98ce\u683c\u8868&#xff09;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u6700\u5173\u952e\u7684\u4e00\u53e5&#xff1a;Spark \u7684 bucket \u53ea\u4fdd\u8bc1&#034;\u4e0d\u9700\u8981\u91cd\u6392&#034;&#xff0c;\u4e0d\u4fdd\u8bc1&#034;\u6570\u636e\u5728\u672c\u5730&#034;\u3002 \u6240\u4ee5\u62ff Doris \u7684 colocate \u76f4\u89c9\u53bb\u8bbe\u8ba1 Spark \u6e56\u8868&#xff0c;\u4f1a\u671f\u5f85\u4e00\u4e2a\u4e0d\u5b58\u5728\u7684\u6536\u76ca\u3002<\/p>\n<p>\u800c\u4e14\u2014\u2014\u6e56\u8868\u8fd9\u4e00\u4fa7\u786e\u5b9e\u6709\u4e00\u4e2a\u5b98\u65b9\u673a\u5236&#xff0c;\u4f46\u5b83\u7684\u540d\u5b57\u548c\u9002\u7528\u8303\u56f4\u4e0e\u591a\u6570\u4eba\u7684\u60f3\u8c61\u4e0d\u540c&#xff1a;<\/p>\n<p>Storage Partition Join&#xff08;SPJ&#xff09;&#xff0c;\u5b98\u65b9\u9010\u5b57&#xff1a;\u201cStorage Partition Join (SPJ) is an optimization technique in Spark SQL that makes use of the existing storage layout to avoid the shuffle phase. This is a generalization of the concept of Bucket Joins, which is only applicable for bucketed tables, to tables partitioned by functions registered in FunctionCatalog. Storage Partition Joins are currently supported for compatible V2 DataSources.\u201d<\/p>\n<p>\u5bf9\u5e94\u5f00\u5173&#xff1a;spark.sql.sources.v2.bucketing.enabled &#061; true&#xff08;3.3.0 \u8d77&#xff09;&#xff0c;\u5b98\u65b9\u63cf\u8ff0 \u201cWhen true, try to eliminate shuffle by using the partitioning reported by a compatible V2 data source.\u201d<\/p>\n<p>\u5b83\u8fd8\u5728\u88ab\u4fee&#xff1a;Spark 4.1.0 \u53d1\u5e03\u8bf4\u660e\u91cc\u6709 [SPARK-53074] Avoid partial clustering in SPJ to meet a child&#039;s required distribution\u3002<\/p>\n<p>\u8fd9\u6bb5\u539f\u6587\u7ea0\u6b63\u4e86\u4e00\u4e2a\u6d41\u4f20\u5f88\u5e7f\u7684\u8bf4\u6cd5\u2014\u2014&#034;\u53ea\u6709 Hive \u5206\u6876\u8868\u624d\u80fd\u514d shuffle&#034;\u5df2\u7ecf\u4e0d\u6210\u7acb&#xff1a;\u5b98\u65b9\u660e\u8bf4 SPJ \u662f Bucket Join \u7684\u6cdb\u5316&#xff0c;\u9002\u7528\u5230&#034;\u6309\u6ce8\u518c\u5728 FunctionCatalog \u91cc\u7684\u51fd\u6570\u5206\u533a&#034;\u7684\u8868&#xff0c;\u800c\u8fd9\u6b63\u662f Iceberg \u9690\u85cf\u5206\u533a&#xff08;days(&#8230;)\u3001bucket(N, col)&#xff09;\u7684\u5f62\u6001\u3002<\/p>\n<p>\u4f46\u5b83\u540c\u65f6\u7ed9\u4e86\u4e24\u4e2a\u9650\u5236\u6761\u4ef6&#xff0c;\u4e24\u4e2a\u90fd\u5f88\u5bb9\u6613\u88ab\u4eba\u8df3\u8fc7&#xff1a;<\/p>\n<p>\u2460 &#034;compatible V2 DataSources&#034;  \u2014\u2014\u517c\u5bb9\u662f\u8fde\u63a5\u5668\u7684\u4e8b&#xff0c;\u4e0d\u662f Spark \u7684\u627f\u8bfa<br \/>\n\u2461 &#034;partitioning reported by&#034;   \u2014\u2014\u8fde\u63a5\u5668\u5fc5\u987b\u771f\u7684\u628a\u5206\u533a\u4fe1\u606f\u4e0a\u62a5\u7ed9 Spark<br \/>\n   \u21d2 \u6240\u4ee5 Iceberg \/ Paimon \u5728\u4f60\u8fd9\u5957\u7248\u672c\u7ec4\u5408\u4e0a\u5230\u5e95\u80fd\u4e0d\u80fd\u5403\u5230 SPJ&#xff0c;<br \/>\n     \u662f\u4e00\u4e2a&#034;\u5b9e\u6d4b\u9879&#034;&#xff0c;\u4e0d\u662f\u4e00\u4e2a&#034;\u80cc\u4e0b\u6765\u7684\u7ed3\u8bba&#034;\u3002\u672c\u6b21\u672a\u53d6\u8bc1\u8fde\u63a5\u5668\u4fa7\u7684\u4e0a\u62a5\u5b9e\u73b0\u3002<\/p>\n<p>\u5b9e\u6d4b\u53ea\u6709\u4e00\u6761\u5224\u636e&#xff08;\u5c31\u662f 5.5.2 \u9a8c\u6536\u91cc\u90a3\u6761&#xff0c;\u8fd9\u91cc\u518d\u5f3a\u8c03\u4e00\u6b21\u5b83\u7684\u666e\u9002\u6027&#xff09;&#xff1a;\u628a join \u4e24\u4fa7\u90fd\u6362\u6210\u6e56\u8868&#xff0c;EXPLAIN FORMATTED \u91cc\u6570 Exchange\u3002\u6d88\u5931\u4e86\u5c31\u662f\u751f\u6548\u4e86&#xff0c;\u6ca1\u6d88\u5931\u5c31\u662f\u6ca1\u751f\u6548\u2014\u2014\u4e0d\u8981\u5f15\u7528\u4efb\u4f55\u4eba\u7684\u535a\u5ba2\u4ee3\u66ff\u8fd9\u4e00\u6b65&#xff0c;\u5305\u62ec\u672c\u7cfb\u5217\u7684\u3002<\/p>\n<p>\u56e0\u6b64\u751f\u4ea7\u4e0a\u7684\u53ef\u884c\u7ec4\u5408\u4ecd\u7136\u662f\u4e09\u6761&#xff0c;\u53ea\u662f\u7b2c\u2460\u6761\u7684\u63aa\u8f9e\u53d8\u4e86&#xff1a;<\/p>\n<p>\u2460 \u70ed\u70b9\u5927\u8868\u4e4b\u95f4\u7684\u9ad8\u9891 join&#xff0c;\u4e14\u80fd\u63a5\u53d7\u91cd\u5237\u4ee3\u4ef7<br \/>\n      \u2192 \u9996\u9009 Hive \u98ce\u683c\u5206\u6876\u8868&#xff08;\u4e24\u4fa7\u540c\u6876\u952e\u540c\u6876\u6570&#xff09;&#xff0c;\u8fd9\u662f\u5b98\u65b9\u660e\u786e\u627f\u8bfa\u7684\u8def\u5f84&#xff1b;<br \/>\n        \u6e56\u8868\u5219\u5148\u5b9e\u6d4b SPJ \u662f\u5426\u751f\u6548&#xff0c;\u751f\u6548\u624d\u5199\u8fdb\u8bbe\u8ba1&#xff0c;\u4e0d\u751f\u6548\u4e0d\u8981\u5f53\u65e2\u6210\u4e8b\u5b9e<br \/>\n\u2461 \u5927\u8868 \u00d7 \u5c0f\u8868<br \/>\n      \u2192 \u4e0d\u8981\u6307\u671b\u5206\u6876\u3002\u8d70\u5e7f\u64ad&#xff08;\u7b2c 4 \u7ae0 BHJ&#xff09;&#xff0c;\u6216\u628a\u5c0f\u8868\u505a\u6210 Broadcast \u53d8\u91cf\/\u4f4d\u56fe<br \/>\n\u2462 \u5927\u8868 \u00d7 \u5927\u8868\u4e14\u90fd\u662f\u6e56\u8868&#xff0c;\u4e14 SPJ \u5b9e\u6d4b\u4e0d\u751f\u6548<br \/>\n      \u2192 \u63a5\u53d7\u4e00\u6b21 shuffle&#xff0c;\u7528 AQE \u63a7\u5e76\u884c\u5ea6 &#043; \u4e24\u9636\u6bb5\u805a\u5408\u7f13\u89e3\u503e\u659c&#xff08;4.5.3&#xff09;<\/p>\n<p>\u8f66\u8054\u7f51\u94fe\u8def\u91cc\u771f\u5b9e\u51fa\u73b0\u7684\u5f62\u6001\u662f&#034;\u8f68\u8ff9\u5927\u8868 \u00d7 \u8f66\u8f86\u65e5\u6c47\u603b\u5927\u8868&#034;&#xff0c;\u4e24\u8005\u90fd\u662f Iceberg\u3002\u8fd9\u6761\u94fe\u8def\u4e0a&#xff0c;\u5206\u6876\u6536\u76ca\u5728\u672a\u5b9e\u6d4b\u524d\u4e00\u5f8b\u6309&#034;\u6ca1\u6709&#034;\u6765\u8bbe\u8ba1\u2014\u2014\u628a SPJ \u5b9e\u6d4b\u5217\u8fdb DDL \u8bc4\u5ba1\u6e05\u5355&#xff0c;\u6bd4\u7559\u4e00\u4e2a&#034;\u7406\u8bba\u4e0a\u80fd\u514d shuffle&#034;\u7684\u671f\u5f85\u6709\u7528\u5f97\u591a\u3002<\/p>\n<h4>5.5.4 \u4e00\u4e2a\u5fc5\u987b\u8bf4\u7684\u53cd\u4f8b&#xff1a;\u6876\u6570\u4e0e\u5e76\u884c\u5ea6\u4e0d\u662f\u4e00\u56de\u4e8b<\/h4>\n<p>   \u8bef\u89e3&#xff1a;8 \u4e2a\u6876 \u21d2 \u67e5\u8be2\u5e76\u884c\u5ea6 8<br \/>\n   \u4e8b\u5b9e&#xff1a;\u8bfb\u4fa7 task \u6570\u7531\u6587\u4ef6\u5206\u7247\u51b3\u5b9a&#xff08;spark.sql.files.maxPartitionBytes\u3001<br \/>\n         openCostInBytes\u3001AQE \u5408\u5e76&#xff09;&#xff0c;\u6876\u53ea\u662f&#034;\u6587\u4ef6\u5f52\u5c5e\u89c4\u5219&#034;<br \/>\n   \u540e\u679c&#xff1a;\u6876\u6570\u8bbe\u6210 8 \u4f46\u6bcf\u6876\u6587\u4ef6\u5f88\u5c0f \u21d2 \u5b9e\u9645\u53ef\u80fd 1 \u4e2a task&#xff1b;<br \/>\n         \u6876\u6570\u8bbe\u6210 8\u3001\u6570\u636e 100GB \u21d2 \u6bcf\u4e2a\u6876\u4e00\u4e2a 12GB \u6587\u4ef6 \u21d2 \u5355 task \u957f\u5c3e&#xff0c;AQE \u6cbb\u4e0d\u4e86\u8bfb\u4fa7&#xff08;4.3 \u8fb9\u754c&#xff09;<\/p>\n<p>\u524d\u4f5c\u5728 SeaTunnel \u4fa7\u6709\u4e00\u6761\u540c\u6784\u7ecf\u9a8c&#xff08;JDBC \u5207\u5206\u5217\u5fc5\u987b\u662f\u7d22\u5f15\u5217&#xff0c;\u5426\u5219\u5e76\u884c SQL \u53cd\u800c\u66f4\u6162&#xff09;\u2014\u2014&#034;\u80fd\u5e76\u884c&#034;\u548c&#034;\u5e76\u884c\u5f97\u6709\u6548&#034;\u662f\u4e24\u4ef6\u4e8b\u3002\u6876\u6570\u3001\u5206\u533a\u6570\u3001shuffle.partitions \u662f\u4e09\u4e2a\u72ec\u7acb\u65cb\u94ae&#xff0c;\u53ea\u6709 5.5.2 \u90a3\u79cd&#034;\u8ba1\u5212\u91cc\u6570 Exchange&#034;\u7684\u5b9e\u6d4b\u80fd\u628a\u5b83\u4eec\u7684\u5173\u7cfb\u786e\u8ba4\u4e0b\u6765\u3002<\/p>\n<hr \/>\n<h3>5.6 \u56db\u5c42\u88c1\u526a\u6f0f\u6597\u7684 Spark \u7248\u672c<\/h3>\n<p>\u5230\u8fd9\u91cc\u53ef\u4ee5\u628a\u56db\u5c42\u91cd\u65b0\u6392\u4e00\u904d&#xff0c;\u5e76\u7ed9\u6bcf\u4e00\u5c42\u4e00\u4e2a\u80fd\u586b\u6570\u5b57\u7684\u683c\u5b50\u3002\u8fd9\u5f20\u56fe\u5c31\u662f ck \u7b2c 4 \u7ae0\u90a3\u5f20\u6f0f\u6597\u7684 Spark \u5bf9\u5e94\u7269\u2014\u2014\u5f62\u72b6\u4e00\u6837&#xff0c;\u540d\u5b57\u4e0d\u540c\u3002<\/p>\n<p>   \u4f60\u7684 SQL \u7684 WHERE \u6761\u4ef6<br \/>\n          \u2502<br \/>\n          \u25bc<br \/>\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n \u2502 \u7b2c 1 \u5c42  \u5206\u533a\u88c1\u526a&#xff08;\u76ee\u5f55 \/ partition spec&#xff09;                      \u2502<br \/>\n \u2502   \u8c01\u6765\u88c1   Spark &#043; \u8868\u683c\u5f0f planner &#043; metastore                    \u2502<br \/>\n \u2502   \u62ff\u4ec0\u4e48\u88c1 \u5206\u533a\u503c&#xff08;Hive\/Paimon&#xff09;\u6216 transform \u57df\u7ea6\u675f&#xff08;Iceberg&#xff09;    \u2502<br \/>\n \u2502   \u89c2\u6d4b     partitions read \/ EXPLAIN \u7684 PartitionFilters \u975e\u7a7a     \u2502<br \/>\n \u2502   \u7701\u6389     \u6574\u4e2a\u76ee\u5f55\u7684 list &#043; \u5143\u6570\u636e &#043; IO        \u2605\u6536\u76ca\u6700\u5927         \u2502<br \/>\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<br \/>\n                            \u25bc  \u5269\u4e0b\u7684\u5019\u9009\u6587\u4ef6<br \/>\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n \u2502 \u7b2c 2 \u5c42  \u6587\u4ef6\u88c1\u526a&#xff08;\u5217\u7edf\u8ba1 lower\/upper bounds\u3001value\/null counts&#xff09;\u2502<br \/>\n \u2502   \u8c01\u6765\u88c1   Iceberg planner&#xff08;\u8bfb manifest \u65f6\u5c31\u88c1&#xff09;\/ Paimon scan    \u2502<br \/>\n \u2502   \u62ff\u4ec0\u4e48\u88c1 \u6bcf\u6587\u4ef6\u6bcf\u5217\u7684\u7edf\u8ba1&#xff08;\u5b57\u7b26\u4e32\u9ed8\u8ba4\u53ea\u8bb0\u524d 16 \u5b57\u7b26&#xff09;           \u2502<br \/>\n \u2502   \u89c2\u6d4b     files read&#xff1a;\u5019\u9009\u6587\u4ef6\u6570 vs \u5b9e\u9645\u6253\u5f00\u6570                  \u2502<br \/>\n \u2502   \u7701\u6389     \u4e0d\u6253\u5f00\u6587\u4ef6&#xff08;\u8fde footer \u90fd\u4e0d\u8bfb&#xff09;                        \u2502<br \/>\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<br \/>\n                            \u25bc  \u6253\u5f00\u7684\u6587\u4ef6<br \/>\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n \u2502 \u7b2c 3 \u5c42  row-group \/ \u5757\u88c1\u526a&#xff08;Parquet column chunk \u7684 min\/max&#xff09;   \u2502<br \/>\n \u2502   \u8c01\u6765\u88c1   Parquet reader&#xff08;\u8c13\u8bcd\u4e0b\u63a8&#xff09;                             \u2502<br \/>\n \u2502   \u5f00\u5173     spark.sql.parquet.filterPushdown &#061; true&#xff08;\u5b98\u65b9\u9ed8\u8ba4&#xff09;    \u2502<br \/>\n \u2502   \u89c2\u6d4b     row groups read&#xff1a;Selected \/ Missing \/ Total           \u2502<br \/>\n \u2502   \u7701\u6389     \u5757\u7ea7 IO&#xff1b;\u5217\u53ea\u8bfb\u9700\u8981\u7684&#xff08;ReadSchema \u5217\u88c1\u526a&#xff09;             \u2502<br \/>\n \u2502   \u2605page \u7ea7&#xff1a;Spark \u5b98\u65b9\u65e0\u5bf9\u5e94\u5f00\u5173&#xff0c;\u672c\u9875\u4e0d\u5199\u6210\u4e8b\u5b9e&#xff08;\u89c1 5.3.1&#xff09;      \u2502<br \/>\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<br \/>\n                            \u25bc  \u8bfb\u8fdb\u5185\u5b58\u7684\u884c<br \/>\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n \u2502 \u7b2c 4 \u5c42  \u884c\u7ea7\u8fc7\u6ee4 &#043; \u5217\u88c1\u526a&#xff08;\u771f\u6b63\u7684 WHERE \u6c42\u503c&#xff09;                  \u2502<br \/>\n \u2502   \u8c01\u6765\u88c1   Spark&#xff08;\u5411\u91cf\u5316\u6279\u5185 \/ \u975e\u5411\u91cf\u5316 record-level filter&#xff09;     \u2502<br \/>\n \u2502   \u89c2\u6d4b     scanned rows \u2192 output rows&#xff1b;bytes read                \u2502<br \/>\n \u2502   \u5230\u8fd9\u4e00\u5c42\u624d&#034;\u7cbe\u786e&#034;&#xff0c;\u4f46\u524d\u4e09\u5c42\u51b3\u5b9a\u4e86\u4f60\u8981\u4ed8\u591a\u5c11\u94b1\u8d70\u5230\u8fd9\u91cc            \u2502<br \/>\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<\/p>\n<p>\u56fe 5-2 \u56db\u5c42\u88c1\u526a\u6f0f\u6597\u4e0e\u6bcf\u5c42\u7684\u89c2\u6d4b\u6307\u6807<\/p>\n<p>\u8868 5-9 \u6bcf\u5c42\u7684\u540d\u5b57\u3001\u5f00\u5173\u4e0e&#034;\u4e0d\u5408\u683c&#034;\u7684\u6837\u5b50<\/p>\n<table>\n<tr>\u5c42Spark \u4fa7\u53eb\u4ec0\u4e48\u4e3b\u8981\u5f00\u5173\/\u524d\u63d0\u4e0d\u5408\u683c\u7684\u5178\u578b\u75c7\u72b6<\/tr>\n<tbody>\n<tr>\n<td>1 \u5206\u533a<\/td>\n<td>PartitionFilters\u3001partitions read<\/td>\n<td>\u8c13\u8bcd\u76f4\u63a5\u547d\u4e2d\u5206\u533a\u5217&#xff08;5.2&#xff09;&#xff1b;Iceberg \u8d70 transform<\/td>\n<td>PartitionFilters: []<\/td>\n<\/tr>\n<tr>\n<td>2 \u6587\u4ef6<\/td>\n<td>files read\u3001metadata tasks<\/td>\n<td>\u5217\u7edf\u8ba1\u6709\u6548&#xff08;\u6392\u5e8f &#043; metrics \u6a21\u5f0f&#xff09;&#xff1b;\u6587\u4ef6\u522b\u592a\u788e<\/td>\n<td>files read \u2248 \u8be5\u67e5\u8be2\u903b\u8f91\u4e0d\u8be5\u78b0\u7684\u6587\u4ef6\u6570<\/td>\n<\/tr>\n<tr>\n<td>3 \u5757<\/td>\n<td>row groups read: Selected\/Missing\/Total<\/td>\n<td>filterPushdown&#061;true&#xff1b;row-group \u5927\u5c0f\u4e0e\u6570\u636e\u76f8\u5173\u6027<\/td>\n<td>Missing: 0&#xff08;\u4e00\u4e2a\u5757\u90fd\u6ca1\u8df3\u8fc7&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>4 \u884c<\/td>\n<td>scanned rows \/ output rows<\/td>\n<td>\u65e0&#xff08;\u5fc5\u7136\u53d1\u751f&#xff09;<\/td>\n<td>scanned rows \/ output rows &gt; 100 \u21d2 \u524d\u4e09\u5c42\u5168\u7a7a\u8f6c<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>5.6.1 \u56db\u6001\u5b9e\u9a8c&#xff1a;\u540c\u4e00\u4efd\u6570\u636e\u6446\u6210\u56db\u79cd\u5e03\u5c40<\/h4>\n<p>\u8fd9\u4e00\u6b65\u662f\u6574\u4e2a\u7cfb\u5217\u7684&#034;\u88c1\u526a\u5929\u82b1\u677f&#034;\u5b9e\u9a8c&#xff0c;\u4e5f\u662f\u672c\u7ae0\u552f\u4e00\u9700\u8981\u4f60\u81ea\u5df1\u586b\u6570\u5b57\u7684\u5730\u65b9\u3002 \u4e4b\u6240\u4ee5\u4e0d\u7ed9\u73b0\u6210\u7ed3\u8bba&#xff0c;\u662f\u56e0\u4e3a\u6587\u4ef6\u6570\u3001\u5757\u6570\u5f3a\u4f9d\u8d56\u7248\u672c\u4e0e\u8fde\u63a5\u5668\u5b9e\u73b0\u2014\u2014\u586b\u8fdb\u53bb\u7684\u6bcf\u4e00\u683c\u90fd\u5fc5\u987b\u662f\u4f60\u5728\u81ea\u5df1\u73af\u5883\u91cc\u91cf\u51fa\u6765\u7684&#xff0c;\u8fd9\u4e5f\u662f\u672c\u7ae0\u53cd\u590d\u5f3a\u8c03\u7684\u7eaa\u5f8b\u3002<\/p>\n<p><span class=\"token comment\">&#8212; \u6587\u4ef6&#xff1a;partition_design.sql \u7684\u7b2c 4 \u6bb5&#xff08;\u56db\u6001\u5bf9\u7167\u8868&#xff09;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u56db\u5f20\u8868\u90fd\u4ece\u7b2c 2 \u7ae0 ODS \u6d3e\u751f&#xff0c;\u6570\u636e\u5b8c\u5168\u76f8\u540c&#xff0c;\u53ea\u6709\u7269\u7406\u5e03\u5c40\u4e0d\u540c<\/span><\/p>\n<p><span class=\"token comment\">&#8212; S0&#xff1a;\u88f8\u8868&#xff08;\u65e0\u5206\u533a\u3001\u65e0\u6392\u5e8f\u3001\u9ed8\u8ba4 512MB \u76ee\u6807&#xff09;<\/span><br \/>\n<span class=\"token keyword\">CREATE<\/span> <span class=\"token keyword\">TABLE<\/span> <span class=\"token keyword\">IF<\/span> <span class=\"token operator\">NOT<\/span> <span class=\"token keyword\">EXISTS<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>t_layout_raw <span class=\"token punctuation\">(<\/span><br \/>\n    vin STRING<span class=\"token punctuation\">,<\/span> city_id <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span> report_date <span class=\"token keyword\">DATE<\/span><span class=\"token punctuation\">,<\/span> reported_at <span class=\"token keyword\">TIMESTAMP<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    speed <span class=\"token keyword\">FLOAT<\/span><span class=\"token punctuation\">,<\/span> battery <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span> battery_pct <span class=\"token keyword\">TINYINT<\/span><span class=\"token punctuation\">,<\/span> vehicle_status STRING<span class=\"token punctuation\">,<\/span> seq <span class=\"token keyword\">BIGINT<\/span><br \/>\n<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">USING<\/span> iceberg TBLPROPERTIES <span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;format-version&#039;<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#039;2&#039;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; S1&#xff1a;\u52a0\u5206\u533a<\/span><br \/>\n<span class=\"token keyword\">CREATE<\/span> <span class=\"token keyword\">TABLE<\/span> <span class=\"token keyword\">IF<\/span> <span class=\"token operator\">NOT<\/span> <span class=\"token keyword\">EXISTS<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>t_layout_part <span class=\"token punctuation\">(<\/span><br \/>\n    vin STRING<span class=\"token punctuation\">,<\/span> city_id <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span> report_date <span class=\"token keyword\">DATE<\/span><span class=\"token punctuation\">,<\/span> reported_at <span class=\"token keyword\">TIMESTAMP<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    speed <span class=\"token keyword\">FLOAT<\/span><span class=\"token punctuation\">,<\/span> battery <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span> battery_pct <span class=\"token keyword\">TINYINT<\/span><span class=\"token punctuation\">,<\/span> vehicle_status STRING<span class=\"token punctuation\">,<\/span> seq <span class=\"token keyword\">BIGINT<\/span><br \/>\n<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">USING<\/span> iceberg PARTITIONED <span class=\"token keyword\">BY<\/span> <span class=\"token punctuation\">(<\/span>days<span class=\"token punctuation\">(<\/span>report_date<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\nTBLPROPERTIES <span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;format-version&#039;<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#039;2&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string\">&#039;write.target-file-size-bytes&#039;<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#039;33554432&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;write.parquet.row-group-size-bytes&#039;<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#039;4194304&#039;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; S2&#xff1a;\u52a0\u5206\u533a &#043; \u8868\u7ea7\u6392\u5e8f&#xff08;\u4e0e S1 \u53ea\u5dee\u4e00\u6761 ALTER \u7684\u65f6\u673a&#xff09;<\/span><br \/>\n<span class=\"token keyword\">CREATE<\/span> <span class=\"token keyword\">TABLE<\/span> <span class=\"token keyword\">IF<\/span> <span class=\"token operator\">NOT<\/span> <span class=\"token keyword\">EXISTS<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>t_layout_sort <span class=\"token punctuation\">(<\/span><br \/>\n    vin STRING<span class=\"token punctuation\">,<\/span> city_id <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span> report_date <span class=\"token keyword\">DATE<\/span><span class=\"token punctuation\">,<\/span> reported_at <span class=\"token keyword\">TIMESTAMP<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    speed <span class=\"token keyword\">FLOAT<\/span><span class=\"token punctuation\">,<\/span> battery <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span> battery_pct <span class=\"token keyword\">TINYINT<\/span><span class=\"token punctuation\">,<\/span> vehicle_status STRING<span class=\"token punctuation\">,<\/span> seq <span class=\"token keyword\">BIGINT<\/span><br \/>\n<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">USING<\/span> iceberg PARTITIONED <span class=\"token keyword\">BY<\/span> <span class=\"token punctuation\">(<\/span>days<span class=\"token punctuation\">(<\/span>report_date<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\nTBLPROPERTIES <span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;format-version&#039;<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#039;2&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string\">&#039;write.target-file-size-bytes&#039;<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#039;33554432&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;write.parquet.row-group-size-bytes&#039;<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#039;4194304&#039;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token keyword\">ALTER<\/span> <span class=\"token keyword\">TABLE<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>t_layout_sort <span class=\"token keyword\">WRITE<\/span> ORDERED <span class=\"token keyword\">BY<\/span> city_id<span class=\"token punctuation\">,<\/span> report_date<span class=\"token punctuation\">,<\/span> vin<span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; S3&#xff1a;\u4e0e S1 \u76f8\u540c\u5e03\u5c40&#xff0c;\u4f46\u4e8b\u540e\u505a Z-Order \u91cd\u5199&#xff08;\u5148\u704c\u6570\u518d rewrite&#xff09;<\/span><br \/>\n<span class=\"token keyword\">CREATE<\/span> <span class=\"token keyword\">TABLE<\/span> <span class=\"token keyword\">IF<\/span> <span class=\"token operator\">NOT<\/span> <span class=\"token keyword\">EXISTS<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>t_layout_zorder <span class=\"token punctuation\">(<\/span><br \/>\n    vin STRING<span class=\"token punctuation\">,<\/span> city_id <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span> report_date <span class=\"token keyword\">DATE<\/span><span class=\"token punctuation\">,<\/span> reported_at <span class=\"token keyword\">TIMESTAMP<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    speed <span class=\"token keyword\">FLOAT<\/span><span class=\"token punctuation\">,<\/span> battery <span class=\"token keyword\">INT<\/span><span class=\"token punctuation\">,<\/span> battery_pct <span class=\"token keyword\">TINYINT<\/span><span class=\"token punctuation\">,<\/span> vehicle_status STRING<span class=\"token punctuation\">,<\/span> seq <span class=\"token keyword\">BIGINT<\/span><br \/>\n<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">USING<\/span> iceberg PARTITIONED <span class=\"token keyword\">BY<\/span> <span class=\"token punctuation\">(<\/span>days<span class=\"token punctuation\">(<\/span>report_date<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\nTBLPROPERTIES <span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;format-version&#039;<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#039;2&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string\">&#039;write.target-file-size-bytes&#039;<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#039;33554432&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;write.parquet.row-group-size-bytes&#039;<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#039;4194304&#039;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p><span class=\"token comment\">&#8212; \u704c\u6570&#xff08;\u56db\u5f20\u8868\u540c\u4e00\u6bb5 SELECT&#xff0c;\u5b57\u6bb5\u5217\u8868\u663e\u5f0f\u5199\u5168&#xff0c;\u907f\u514d\u5217\u5e8f\u4e32\u4f4d&#xff09;<\/span><br \/>\n<span class=\"token keyword\">INSERT<\/span> <span class=\"token keyword\">INTO<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>t_layout_raw<br \/>\n<span class=\"token keyword\">SELECT<\/span> vin<span class=\"token punctuation\">,<\/span> city_id<span class=\"token punctuation\">,<\/span> report_date<span class=\"token punctuation\">,<\/span> reported_at<span class=\"token punctuation\">,<\/span> speed<span class=\"token punctuation\">,<\/span> battery<span class=\"token punctuation\">,<\/span> battery_pct<span class=\"token punctuation\">,<\/span> vehicle_status<span class=\"token punctuation\">,<\/span> seq<br \/>\n<span class=\"token keyword\">FROM<\/span> paimon<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>ods_vehicle_telemetry<span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u5176\u4f59\u4e09\u5f20\u8868\u91cd\u590d\u540c\u4e00\u6bb5&#xff0c;\u53ea\u5728 t_layout_zorder \u704c\u5b8c\u540e\u6267\u884c&#xff1a;<\/span><br \/>\n<span class=\"token keyword\">CALL<\/span> iceberg<span class=\"token punctuation\">.<\/span>system<span class=\"token punctuation\">.<\/span>rewrite_data_files<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token keyword\">table<\/span> <span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token string\">&#039;demo.t_layout_zorder&#039;<\/span><span class=\"token punctuation\">,<\/span> strategy <span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token string\">&#039;sort&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    sort_order <span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token string\">&#039;zorder(city_id, report_date)&#039;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff1a;\u4e3a\u4ec0\u4e48 S0 \u4e0d\u8bbe\u5c0f target\u3001S1~S3 \u8bbe\u6210 32MB\/4MB&#xff1f;\u56e0\u4e3a\u5bf9\u7167\u5b9e\u9a8c\u5fc5\u987b\u8ba9&#034;\u5e03\u5c40&#034;\u6210\u4e3a\u552f\u4e00\u53d8\u91cf\u3002S0 \u4e00\u4e2a 512MB \u76ee\u6807\u4e0b\u53ea\u6709 1 \u4e2a\u6587\u4ef6&#xff0c;\u5b83\u7684 files read \u6052\u4e3a 1\u2014\u2014\u8fd9\u672c\u8eab\u5c31\u662f\u6700\u6709\u8bf4\u670d\u529b\u7684\u4e00\u683c&#xff1a;\u6587\u4ef6\u90fd\u53ea\u6709\u4e00\u4e2a&#xff0c;\u8c08\u4e0d\u4e0a\u6587\u4ef6\u7ea7\u88c1\u526a\u3002<\/p>\n<p>\u8f93\u51fa\u8868\u7ed3\u6784&#xff1a;\u56db\u5f20\u8868\u5217\u7ed3\u6784\u5b8c\u5168\u4e00\u81f4&#xff08;9 \u5217&#xff09;&#xff0c;\u884c\u6570\u5b8c\u5168\u4e00\u81f4&#xff08;200000&#xff09;\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6&#xff08;\u4e09\u6761\u5fc5\u987b\u5168\u8fc7&#xff0c;\u5426\u5219\u56db\u6001\u5bf9\u6bd4\u65e0\u6548&#xff09;<\/p>\n<p><span class=\"token keyword\">SELECT<\/span> <span class=\"token string\">&#039;raw&#039;<\/span> <span class=\"token keyword\">AS<\/span> st<span class=\"token punctuation\">,<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> c <span class=\"token keyword\">FROM<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>t_layout_raw<br \/>\n<span class=\"token keyword\">UNION<\/span> <span class=\"token keyword\">ALL<\/span> <span class=\"token keyword\">SELECT<\/span> <span class=\"token string\">&#039;part&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>t_layout_part<br \/>\n<span class=\"token keyword\">UNION<\/span> <span class=\"token keyword\">ALL<\/span> <span class=\"token keyword\">SELECT<\/span> <span class=\"token string\">&#039;sort&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>t_layout_sort<br \/>\n<span class=\"token keyword\">UNION<\/span> <span class=\"token keyword\">ALL<\/span> <span class=\"token keyword\">SELECT<\/span> <span class=\"token string\">&#039;zorder&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>t_layout_zorder<span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; \u56db\u884c\u5fc5\u987b\u5b8c\u5168\u76f8\u7b49<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token string\">&#039;zorder&#039;<\/span> <span class=\"token keyword\">AS<\/span> st<span class=\"token punctuation\">,<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> f<span class=\"token punctuation\">,<\/span> <span class=\"token function\">sum<\/span><span class=\"token punctuation\">(<\/span>record_count<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">AS<\/span> r <span class=\"token keyword\">FROM<\/span> iceberg<span class=\"token punctuation\">.<\/span>demo<span class=\"token punctuation\">.<\/span>t_layout_zorder<span class=\"token punctuation\">.<\/span>files<span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; r \u5fc5\u987b\u7b49\u4e8e 200000&#xff08;rewrite \u4e0d\u5220\u884c&#xff09;<\/span><\/p>\n<p>\u4e24\u4e2a\u63a2\u9488\u67e5\u8be2&#xff0c;\u5206\u522b\u4ee3\u8868\u4e24\u7c7b\u771f\u5b9e\u9700\u6c42&#xff1a;<\/p>\n<p><span class=\"token comment\">&#8212; Q1&#xff1a;\u9700\u6c42 A \u5f62\u6001&#xff08;\u65f6\u95f4 &#043; \u57ce\u5e02\u90fd\u6709&#xff09;\u2014\u2014\u5206\u533a\u5c31\u8be5\u5e2e\u5fd9<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> <span class=\"token operator\">&lt;<\/span>state_table<span class=\"token operator\">&gt;<\/span> <span class=\"token keyword\">WHERE<\/span> report_date <span class=\"token operator\">&#061;<\/span> <span class=\"token keyword\">DATE<\/span> <span class=\"token string\">&#039;2026-10-15&#039;<\/span> <span class=\"token operator\">AND<\/span> city_id <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">310007<\/span><span class=\"token punctuation\">;<\/span><br \/>\n<span class=\"token comment\">&#8212; Q2&#xff1a;\u9700\u6c42 B \u5f62\u6001&#xff08;\u53ea\u7ed9\u57ce\u5e02&#xff0c;\u8de8\u5168\u6708&#xff09;\u2014\u2014\u5206\u533a\u65e0\u80fd\u4e3a\u529b&#xff0c;\u53ea\u80fd\u9760\u7b2c 2\/3 \u5c42<\/span><br \/>\n<span class=\"token keyword\">SELECT<\/span> <span class=\"token function\">count<\/span><span class=\"token punctuation\">(<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">FROM<\/span> <span class=\"token operator\">&lt;<\/span>state_table<span class=\"token operator\">&gt;<\/span> <span class=\"token keyword\">WHERE<\/span> city_id <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">310007<\/span><span class=\"token punctuation\">;<\/span><\/p>\n<h4>5.6.2 \u91cf\u51fa\u6765&#xff1a;measure_scan.py<\/h4>\n<p>Spark \u7684 EXPLAIN \u4e0d\u7ed9\u5b9e\u6d4b&#xff08;\u7b2c 4 \u7ae0 4.2 \u90a3\u6761\u4e0d\u5bf9\u79f0&#xff09;&#xff0c;\u5b9e\u6d4b\u53ea\u80fd\u4ece\u4e8b\u4ef6\u65e5\u5fd7\u62ff\u3002\u7b2c 2 \u7ae0\u5df2\u7ecf\u628a spark.eventLog.enabled \u6253\u5f00\u4e86&#xff0c;\u8fd9\u91cc\u76f4\u63a5\u7528\u5b83\u2014\u2014\u8fd9\u5c31\u662f\u4e3a\u4ec0\u4e48\u4e8b\u4ef6\u65e5\u5fd7\u8981\u5728\u642d\u73af\u5883\u7684\u7b2c\u4e00\u4e2a\u5c0f\u65f6\u5c31\u5f00&#xff0c;\u800c\u4e0d\u662f\u7b49\u5230\u7b2c\u4e00\u6b21\u6392\u969c\u3002<\/p>\n<p><span class=\"token comment\"># \u6587\u4ef6&#xff1a;measure_scan.py<\/span><br \/>\n<span class=\"token comment\"># \u7528\u6cd5&#xff1a;spark.eventLog.enabled&#061;true \u8dd1\u5b8c\u67e5\u8be2\u540e&#xff0c;\u5bf9\u751f\u6210\u7684\u4e8b\u4ef6\u65e5\u5fd7\u6267\u884c<\/span><br \/>\n<span class=\"token comment\">#       python measure_scan.py \/data\/spark\/eventlog\/local-xxx.log<\/span><br \/>\n<span class=\"token comment\"># \u4f9d\u8d56&#xff1a;\u4ec5\u6807\u51c6\u5e93<\/span><br \/>\n<span class=\"token keyword\">import<\/span> json<span class=\"token punctuation\">,<\/span> sys<span class=\"token punctuation\">,<\/span> re<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">collect<\/span><span class=\"token punctuation\">(<\/span>path<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u4ece\u4e8b\u4ef6\u65e5\u5fd7\u91cc\u6293\u6bcf\u4e2a SQL execution \u7684\u7d2f\u52a0\u5668\u6307\u6807\u3002&#034;&#034;&#034;<\/span><br \/>\n    accs<span class=\"token punctuation\">,<\/span> sqls <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> line <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">open<\/span><span class=\"token punctuation\">(<\/span>path<span class=\"token punctuation\">,<\/span> encoding<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;utf-8&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">try<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            e <span class=\"token operator\">&#061;<\/span> json<span class=\"token punctuation\">.<\/span>loads<span class=\"token punctuation\">(<\/span>line<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">except<\/span> json<span class=\"token punctuation\">.<\/span>JSONDecodeError<span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">continue<\/span><br \/>\n        t <span class=\"token operator\">&#061;<\/span> e<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;Event&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">if<\/span> t <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#034;SparkListenerSQLExecutionStart&#034;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            sqls<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>e<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;executionId&#034;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> e<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;description&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">elif<\/span> t <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#034;SparkListenerSQLAccumulator&#034;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            m <span class=\"token operator\">&#061;<\/span> e<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;metric&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">or<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><br \/>\n            acc <span class=\"token operator\">&#061;<\/span> e<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;accumualtorUpdate&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">or<\/span> e<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;accumulatorUpdate&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">or<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><br \/>\n            name <span class=\"token operator\">&#061;<\/span> m<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;name&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">or<\/span> <span class=\"token string\">&#034;&#034;<\/span><br \/>\n            typ  <span class=\"token operator\">&#061;<\/span> m<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;type&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">or<\/span> <span class=\"token string\">&#034;&#034;<\/span><br \/>\n            val  <span class=\"token operator\">&#061;<\/span> m<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;value&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> val <span class=\"token keyword\">is<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                val <span class=\"token operator\">&#061;<\/span> acc<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span>e<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;accumualtorId&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            key <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>e<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;sqlExecutionId&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> name<span class=\"token punctuation\">,<\/span> typ<span class=\"token punctuation\">)<\/span><br \/>\n            accs<span class=\"token punctuation\">[<\/span>key<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> accs<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span>key<span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token punctuation\">(<\/span>val <span class=\"token keyword\">if<\/span> <span class=\"token builtin\">isinstance<\/span><span class=\"token punctuation\">(<\/span>val<span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token builtin\">int<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token builtin\">float<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">else<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> sqls<span class=\"token punctuation\">,<\/span> accs<\/p>\n<p><span class=\"token comment\"># \u5173\u5fc3\u7684\u6307\u6807\u540d&#xff08;\u4e0d\u540c\u7248\u672c\u547d\u540d\u7565\u6709\u5dee\u5f02&#xff1a;\u5148\u8dd1\u4e00\u6b21\u770b\u5b83\u5230\u5e95\u53eb\u4ec0\u4e48&#xff09;<\/span><br \/>\nPATTERN <span class=\"token operator\">&#061;<\/span> re<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">compile<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">r&#034;(file|row group|page|scan|record|bytes|partitions)&#034;<\/span><span class=\"token punctuation\">,<\/span> re<span class=\"token punctuation\">.<\/span>I<span class=\"token punctuation\">)<\/span><\/p>\n<p>sqls<span class=\"token punctuation\">,<\/span> accs <span class=\"token operator\">&#061;<\/span> collect<span class=\"token punctuation\">(<\/span>sys<span class=\"token punctuation\">.<\/span>argv<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">for<\/span> eid<span class=\"token punctuation\">,<\/span> desc <span class=\"token keyword\">in<\/span> sqls<span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\\\\n&#061;&#061;&#061; SQL <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>eid<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> :: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>desc<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">110]<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> <span class=\"token punctuation\">(<\/span>seid<span class=\"token punctuation\">,<\/span> name<span class=\"token punctuation\">,<\/span> typ<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> v <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">sorted<\/span><span class=\"token punctuation\">(<\/span>accs<span class=\"token punctuation\">.<\/span>items<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">if<\/span> seid <span class=\"token operator\">&#061;&#061;<\/span> eid <span class=\"token keyword\">and<\/span> PATTERN<span class=\"token punctuation\">.<\/span>search<span class=\"token punctuation\">(<\/span>name <span class=\"token keyword\">or<\/span> <span class=\"token string\">&#034;&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;    <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>name<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">&lt;38<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>typ<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">&lt;12<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>v<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u53c2\u6570\u8bf4\u660e&#xff1a;\u4e8b\u4ef6\u65e5\u5fd7\u91cc SQL \u6307\u6807\u7684\u5b57\u6bb5\u540d\u5728\u5404\u7248\u672c\u95f4\u786e\u6709\u5dee\u5f02&#xff08;accumualtorUpdate \u8fd9\u4e2a\u62fc\u5199\u662f\u5b98\u65b9\u4e8b\u4ef6\u65e5\u5fd7\u91cc\u5386\u53f2\u9057\u7559\u7684\u7b14\u8bef&#xff0c;\u811a\u672c\u5bf9\u65b0\u65e7\u4e24\u79cd\u5199\u6cd5\u90fd\u505a\u4e86\u517c\u5bb9\u5c1d\u8bd5\u2014\u2014\u8fd9\u4e5f\u662f&#034;\u65e5\u5fd7\u683c\u5f0f\u4e0d\u8981\u9760\u731c&#034;\u7684\u4e00\u4e2a\u5177\u4f53\u4f8b\u5b50&#xff09;\u3002\u6240\u4ee5\u811a\u672c\u7684\u7b56\u7565\u662f\u5148\u628a\u4f60\u73af\u5883\u91cc\u6240\u6709\u540d\u5b57\u91cc\u5e26 file\/row group\/page\/scan\/record\/bytes \u7684\u6307\u6807\u5168\u6253\u51fa\u6765&#xff0c;\u518d\u4eba\u5de5\u8ba4\u54ea\u4e00\u4e2a\u662f&#034;\u6587\u4ef6\u6570&#034;\u3002\u8fd9\u662f\u6709\u610f\u7684&#xff1a;\u80cc\u6307\u6807\u540d\u4e0d\u5982\u8ba4\u51fa\u6307\u6807\u540d\u3002<\/p>\n<p>\u8f93\u51fa\u7ed3\u6784&#xff1a;\u6bcf\u6761 SQL \u4e00\u6bb5&#xff0c;\u9010\u884c \u6307\u6807\u540d \/ \u7c7b\u578b \/ \u7d2f\u52a0\u503c\u3002<\/p>\n<p>\u9a8c\u6536\u6807\u51c6&#xff1a;\u80fd\u7a33\u5b9a\u6253\u51fa\u56db\u4e2a\u6570\u5b57\u2014\u2014\u6587\u4ef6\u6570\u3001row-group \u6570&#xff08;Selected\/Total \u6216\u7b49\u4ef7\u9879&#xff09;\u3001scanned rows\u3001bytes read\u3002\u56db\u4e2a\u90fd\u62ff\u4e0d\u5230\u5c31\u5148\u53bb\u7b2c 17 \u7ae0&#xff08;\u53ef\u89c2\u6d4b\u843d\u5730&#xff09;&#xff0c;\u522b\u7528\u4f30\u7b97\u6570\u5b57\u4ee3\u66ff\u5b9e\u6d4b\u3002<\/p>\n<h4>5.6.3 \u8bb0\u5f55\u8868&#xff1a;\u628a\u5b83\u586b\u6210\u56e2\u961f\u7684\u7b2c\u4e00\u4efd\u57fa\u7ebf<\/h4>\n<p>\u8868 5-10 \u56db\u6001\u5b9e\u6d4b\u8bb0\u5f55&#xff08;\u672c\u7cfb\u5217\u4e0d\u4ee3\u586b\u6570\u5b57&#xff09;<\/p>\n<table>\n<tr>\u72b6\u6001\u67e5\u8be2partitions readfiles readrow groups Selected\/Totalscanned rowsbytes read\u7ed3\u8bba<\/tr>\n<tbody>\n<tr>\n<td>S0 \u88f8\u8868<\/td>\n<td>Q1<\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td>\u57fa\u7ebf&#xff1a;\u5168\u626b<\/td>\n<\/tr>\n<tr>\n<td>S1 &#043;\u5206\u533a<\/td>\n<td>Q1<\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td>\u9884\u671f \u2248 S0 \u7684 1\/30<\/td>\n<\/tr>\n<tr>\n<td>S2 &#043;\u5206\u533a&#043;\u6392\u5e8f<\/td>\n<td>Q1<\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td>\u7b2c 2\/3 \u5c42\u5f00\u59cb\u6709 Missing<\/td>\n<\/tr>\n<tr>\n<td>S3 &#043;\u5206\u533a&#043;Z-Order<\/td>\n<td>Q1<\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td>\u4e0e S2 \u63a5\u8fd1\u6216\u7565\u5dee&#xff08;\u65f6\u95f4\u5df2\u5355\u72ec\u547d\u4e2d&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>S0 \u88f8\u8868<\/td>\n<td>Q2<\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td>\u5168\u626b&#xff0c;\u4e14\u6587\u4ef6\u5c11 \u21d2 \u5757\u5f88\u5bbd<\/td>\n<\/tr>\n<tr>\n<td>S1 &#043;\u5206\u533a<\/td>\n<td>Q2<\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td>\u5206\u533a\u5e2e\u4e0d\u4e0a&#xff08;\u65e0\u65e5\u671f\u6761\u4ef6&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>S2 &#043;\u5206\u533a&#043;\u6392\u5e8f<\/td>\n<td>Q2<\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td>\u6392\u5e8f\u9996\u5217\u662f city \u21d2 \u5e94\u663e\u8457\u4e0b\u964d<\/td>\n<\/tr>\n<tr>\n<td>S3 &#043;\u5206\u533a&#043;Z-Order<\/td>\n<td>Q2<\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<td>Z-Order \u7684\u4e3b\u573a<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u586b\u5b8c\u540e\u4f60\u4f1a\u5f97\u5230\u4e09\u4e2a\u7ed3\u8bba&#xff0c;\u5b83\u4eec\u6bd4\u4efb\u4f55\u4e00\u6bb5\u8bf4\u660e\u90fd\u6709\u7528&#xff1a;<\/p>\n<p>\u2460 S1 \u5bf9 Q1 \u6709\u6548\u3001\u5bf9 Q2 \u65e0\u6548      \u21d2 \u5206\u533a\u53ea\u88c1\u5b83\u5207\u7684\u90a3\u4e2a\u7ef4\u5ea6&#xff0c;\u522b\u6307\u671b\u5b83\u987a\u624b\u88c1\u522b\u7684<br \/>\n\u2461 S2 \u4e0e S1 \u7684\u5dee&#xff0c;\u5c31\u662f&#034;\u6392\u5e8f&#034;\u7684\u51c0\u503c \u21d2 \u51c0\u503c\u4e3a 0 \u65f6&#xff0c;\u8bf4\u660e\u6587\u4ef6\u592a\u5bbd\u6216\u7edf\u8ba1\u592a\u7cd9&#xff08;\u56de 5.3.3\/5.3.4&#xff09;<br \/>\n\u2462 S3 \u4e0e S2 \u7684\u76f8\u5bf9\u9ad8\u4f4e\u51b3\u5b9a\u9009\u578b     \u21d2 \u4f60\u7684\u771f\u5b9e\u67e5\u8be2\u91cc&#034;\u53ea\u7ed9 city&#034;\u5360\u591a\u5c11&#xff1f;<br \/>\n                                     \u5360\u6bd4\u4f4e\u5c31\u7559\u7ebf\u6027&#xff0c;\u522b\u4e3a\u4e86 Q2 \u727a\u7272 Q1 \u548c\u6d41\u5f0f\u5bfc\u51fa<\/p>\n<p>\u5b9e\u9a8c\u89c4\u6a21\u4e0d\u8db3\u65f6\u7684\u73b0\u8c61\u8981\u63d0\u524d\u77e5\u9053&#xff1a;20 \u4e07\u884c\u6570\u636e\u5373\u4f7f\u5207\u6210 32MB \u4e5f\u53ea\u6709 1~2 \u4e2a\u6587\u4ef6&#xff0c;S2\/S3 \u7684\u5dee\u522b\u53ef\u80fd\u91cf\u4e0d\u51fa\u6765\u3002\u8fd9\u4e0d\u662f\u5b9e\u9a8c\u5931\u8d25&#xff0c;\u662f\u4f60\u7684\u6570\u636e\u6ca1\u5230\u9700\u8981\u88c1\u526a\u7684\u89c4\u6a21\u3002 \u60f3\u770b\u5230\u5dee\u522b&#xff0c;\u628a\u7b2c 2 \u7ae0\u7684 sequence(1, 200000) \u6539\u6210 sequence(1, 20000000) \u91cd\u704c\u4e00\u904d&#xff08;\u9020\u6570\u539f\u5219\u4e0d\u53d8&#xff1a;\u5168\u90e8\u4ece v \u6d3e\u751f&#xff0c;\u7981\u6b62 rand()&#xff09;&#xff0c;\u6216\u8005\u628a target \u8fdb\u4e00\u6b65\u964d\u5230 4MB\u3002\u8bb0\u4f4f\u65b9\u5411&#xff1a;\u5b9e\u9a8c\u65f6\u628a\u6587\u4ef6\u5207\u5c0f\u662f\u4e3a\u4e86\u770b\u6e05\u673a\u5236&#xff1b;\u751f\u4ea7\u65f6\u628a\u6587\u4ef6\u5408\u5927\u624d\u662f\u7701\u94b1\u3002<\/p>\n<hr \/>\n<h3>\u5de5\u7a0b\u8981\u70b9<\/h3>\n<li>\n<p>\u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u662f\u4e09\u4ef6\u4e8b&#xff0c;\u5206\u522b\u51b3\u5b9a&#034;\u54ea\u4e9b\u76ee\u5f55&#034;\u201c\u54ea\u4e9b\u6587\u4ef6\u201d\u201c\u6587\u4ef6\u5185\u7684\u5757\u201d\u3002 \u6df7\u8c08\u5b83\u4eec\u7684\u540e\u679c\u662f&#xff1a;\u7528\u6392\u5e8f\u8865\u5206\u533a\u7684\u6d1e&#xff0c;\u6216\u7528\u6876\u6570\u5f53\u5e76\u884c\u5ea6\u8c03\u3002\u4e09\u5c42\u7684\u6536\u76ca\u4e0d\u53ef\u4e92\u66ff\u3002<\/p>\n<\/li>\n<li>\n<p>Hive \u76ee\u5f55\u5206\u533a\u7531 Spark \u88c1&#xff0c;Iceberg \u9690\u85cf\u5206\u533a\u7531\u8868\u683c\u5f0f planner \u88c1&#xff0c;Paimon \u7531 scan \u6309\u5206\u533a &#043; \u6876\u4e24\u7ea7\u5b9a\u4f4d\u3002 \u8c13\u8bcd\u5199\u6cd5\u5fc5\u987b\u8ddf\u7740&#034;\u8c01\u6765\u88c1&#034;\u53d8\u2014\u2014\u7ed9 Iceberg \u8868\u5199\u5206\u533a\u76ee\u5f55\u540d&#xff0c;\u7ed9 Paimon \u8868\u5bf9 bucket-key \u5199\u8303\u56f4\u6761\u4ef6&#xff0c;\u90fd\u662f\u767d\u5199\u3002<\/p>\n<\/li>\n<li>\n<p>PartitionFilters \u975e\u7a7a\u4e0d\u7b49\u4e8e\u5c11\u8bfb\u6570\u636e\u3002 \u6392\u9664\u5f0f\u8c13\u8bcd&#xff08;&lt;&gt;\u3001NOT IN&#xff09;\u901a\u5e38&#034;\u80fd\u88c1\u4f46\u88c1\u4e0d\u52a8&#034;\u3002&#034;\u88c1\u4e86\u591a\u5c11&#034;\u53ea\u7531 files read \u8bf4\u8bdd\u3002<\/p>\n<\/li>\n<li>\n<p>\u516d\u4e2a\u5931\u6548\u91cc\u552f\u4e00\u4e0d\u8868\u73b0\u4e3a\u6162\u7684\u662f&#034;\u5206\u533a\u503c\u683c\u5f0f\u4e0d\u4e00\u81f4&#034;&#xff0c;\u5b83\u8868\u73b0\u4e3a\u7a7a\u7ed3\u679c\u3002 \u8de8\u5f15\u64ce\u5199 __DT__ \u65f6\u8fd9\u662f\u9996\u8981\u98ce\u9669&#xff0c;\u5904\u7f6e\u662f\u56fa\u5b9a\u683c\u5f0f &#043; raise_error \u884c\u6570\u65ad\u8a00&#xff0c;\u4e0e\u7248\u672c\u65e0\u5173\u3001\u4e0e\u5b9e\u6d4b\u65e0\u5173&#xff0c;\u4e00\u5f8b\u52a0\u3002<\/p>\n<\/li>\n<li>\n<p>\u8c13\u8bcd\u4e0b\u63a8\u751f\u6548 \u2260 \u6570\u636e\u88ab\u88c1\u6389\u3002 \u4e0b\u63a8\u53ea\u4fdd\u8bc1\u6761\u4ef6\u8fdb\u4e86 reader&#xff1b;\u80fd\u4e0d\u80fd\u8df3\u5757\u53d6\u51b3\u4e8e\u7edf\u8ba1\u8fb9\u754c\u7a84\u4e0d\u7a84\u3002\u5224\u636e\u770b scanned rows \/ output rows \u7684\u6bd4\u503c&#xff0c;&gt;100 \u5c31\u662f\u524d\u4e09\u5c42\u5168\u7a7a\u8f6c\u3002<\/p>\n<\/li>\n<li>\n<p>\u5b57\u7b26\u4e32\u5217\u7684 min\/max \u9ed8\u8ba4\u53ea\u8bb0\u524d 16 \u5b57\u7b26&#xff08;Iceberg write.metadata.metrics.default &#061; truncate(16)&#xff09;\u3002 VIN\u3001\u62a5\u6587\u3001URL \u8fd9\u7c7b&#034;\u524d\u7f00\u957f\u3001\u53d8\u5316\u77ed&#034;\u7684\u5217&#xff0c;\u6392\u5e8f\u505a\u5f97\u518d\u597d\u4e5f\u88c1\u4e0d\u52a8\u2014\u2014\u9ad8\u4ef7\u503c\u5217\u663e\u5f0f\u8bbe full&#xff0c;\u4ee3\u4ef7\u662f\u5143\u6570\u636e\u53d8\u5927\u3002<\/p>\n<\/li>\n<li>\n<p>\u5411\u91cf\u5316 reader \u4e0e Parquet record-level filtering \u662f\u4e92\u65a5\u8def\u5f84&#xff08;\u5b98\u65b9\u539f\u6587&#xff1a;\u540e\u8005&#034;\u53ea\u5728\u5411\u91cf\u8bfb\u53d6\u5668\u672a\u4f7f\u7528\u65f6\u624d\u6709\u6548&#034;&#xff09;\u3002\u522b\u628a\u5b83\u4eec\u5f53\u4e24\u4e2a\u53e0\u52a0\u7684\u5f00\u5173&#xff1b;\u6362\u8def\u5f84\u5fc5\u987b\u5b9e\u6d4b&#xff0c;\u5224\u636e\u662f Batched: true\/false \u4e0e scanned rows\u3002<\/p>\n<\/li>\n<li>\n<p>\u5c0f\u6587\u4ef6\u6709\u4e09\u7b14\u72ec\u7acb\u7684\u8d26&#xff1a;\u6253\u5f00\u6210\u672c\u3001planning \u6210\u672c\u3001\u89e3\u7801\u6210\u672c\u3002 \u53ea\u76ef&#034;\u6587\u4ef6\u591a&#034;\u4f1a\u6f0f\u6389\u540e\u4e24\u7b14\u3002Iceberg rewrite_data_files \u9ed8\u8ba4\u53ea\u5904\u7406 &lt;75% target \u4e14\u540c\u7ec4 \u22655 \u4e2a\u7684\u6587\u4ef6\u2014\u2014&#034;\u8dd1\u4e86\u6ca1\u6548\u679c&#034;\u591a\u534a\u662f\u9ed8\u8ba4\u53c2\u6570&#xff0c;\u4e0d\u662f bug\u3002<\/p>\n<\/li>\n<li>\n<p>Spark \u5b98\u65b9 INSERT \u8bed\u6cd5\u91cc\u6ca1\u6709&#034;\u5199\u5165\u65f6\u6392\u5e8f&#034;\u3002 \u6392\u5e8f\u5199\u5165\u662f connector \u901a\u8fc7 RequiresDistributionAndOrdering \u7533\u8bf7\u7684\u3002\u8bed\u6cd5\u8981\u80cc\u7684\u662f ALTER TABLE &#8230; WRITE ORDERED BY \/ WRITE LOCALLY ORDERED BY \/ WRITE UNORDERED \u548c rewrite_data_files&#xff0c;\u4e0d\u662f INSERT &#8230; ORDER BY\u3002<\/p>\n<\/li>\n<li>\n<p>WRITE ORDERED BY \u53ea\u7ba1\u65b0\u5199\u5165\u548c MERGE INTO&#xff0c;\u5386\u53f2\u6587\u4ef6\u4e00\u884c\u90fd\u4e0d\u4f1a\u91cd\u6392\u3002 \u52a0\u4e86\u6392\u5e8f\u53d1\u73b0\u6ca1\u53d8\u5feb&#xff0c;\u7b2c\u4e00\u4ef6\u4e8b\u662f\u67e5\u6709\u6ca1\u6709 rewrite \u8fc7\u3002<\/p>\n<\/li>\n<li>\n<p>\u6392\u5e8f\u5217\u6570\u6309\u5b98\u65b9\u533a\u95f4\u9009&#xff1a;1 \u5217\u7ebf\u6027\u30012~4 \u5217 Z-Order\u30015 \u5217\u4ee5\u4e0a Hilbert\u3002 \u8d85\u8fc7\u56db\u5217\u7ee7\u7eed\u5f80 Z-Order \u585e\u5df2\u7ecf\u8d8a\u51fa\u5efa\u8bae\u8303\u56f4&#xff1b;\u4e24\u4e2a\u7ef4\u5ea6\u90fd\u60f3\u6700\u4f18\u7684\u8868\u4e0d\u5b58\u5728&#xff0c;\u8be5\u62c6\u8868\u5c31\u62c6\u8868\u3002<\/p>\n<\/li>\n<li>\n<p>Z-Order \u6709\u4e24\u7b14\u9690\u6027\u4ee3\u4ef7&#xff1a;\u4f1a\u88ab\u8ffd\u52a0\u5199\u5165\u7a00\u91ca&#xff08;\u53d8\u6210\u7b2c 10 \u7ae0\u7684\u957f\u671f\u7ef4\u62a4\u9879&#xff09;&#xff0c;\u5e76\u4e14\u4f1a\u7834\u574f\u6309\u539f\u952e\u7684\u6709\u5e8f\u6027&#xff08;\u6d41\u5f0f\u5bfc\u51fa\u3001\u7a97\u53e3\u4f5c\u4e1a\u88ab\u8feb\u91cd\u6392&#xff09;\u3002 \u6536\u76ca\u8981\u62ff\u5b9e\u6d4b\u6362&#xff0c;\u522b\u62ff\u76f4\u89c9\u6362\u3002<\/p>\n<\/li>\n<li>\n<p>Iceberg 1.12 \u7684 rewrite_data_files \u53ea\u6709 strategy \/ sort_order \/ where \/ options\u3002 \u65e7\u53c2\u6570\u540d sort_by \/ zorder_by \/ distribute_by \/ sort_buffer_size \u5728\u8fd9\u4e00\u7248\u6587\u6863\u91cc\u51fa\u73b0\u6b21\u6570\u4e3a 0&#xff0c;\u7167\u65e7\u5199\u4f1a\u76f4\u63a5\u62a5\u9519&#xff1b;sort_by \u53ea\u5b58\u5728\u4e8e rewrite_manifests&#xff0c;\u8bed\u4e49\u5b8c\u5168\u4e0d\u540c\u3002<\/p>\n<\/li>\n<li>\n<p>&#034;\u53ea\u6709 Hive \u5206\u6876\u8868\u80fd\u514d shuffle&#034;\u5df2\u7ecf\u4e0d\u6210\u7acb&#xff0c;\u4f46\u4e5f\u4e0d\u7b49\u4e8e\u6e56\u8868\u80fd\u514d\u3002 \u5b98\u65b9 SPJ \u662f Bucket Join \u7684\u6cdb\u5316\u3001\u9762\u5411&#034;\u517c\u5bb9\u7684 V2 \u6570\u636e\u6e90&#034;&#xff0c;\u517c\u5bb9\u4e0e\u4e0a\u62a5\u90fd\u662f\u8fde\u63a5\u5668\u7684\u8d23\u4efb\u3002\u552f\u4e00\u5224\u636e\u662f join \u4e24\u4fa7 Exchange \u6709\u6ca1\u6709\u6d88\u5931&#xff0c;\u5b9e\u6d4b\u4e00\u6b21&#xff0c;\u5b58\u6863\u4e00\u6b21\u3002<\/p>\n<\/li>\n<li>\n<p>Paimon \u5b98\u65b9\u4e00\u53e5\u8bdd\u5426\u6389\u4e86\u6700\u5e38\u89c1\u7684\u8bef\u89e3&#xff1a;\u201cbucket count is not a hard read-parallelism cap.\u201d \u6876\u6570\u6309&#034;\u5355\u6876 200MB\u20131GB&#034;\u8d77\u6b65&#xff08;\u5b98\u65b9\u660e\u8bf4\u8fd9\u662f guide \u4e0d\u662f limit&#xff09;&#xff0c;\u8bfb\u5e76\u884c\u5ea6\u7531\u6587\u4ef6\u5206\u7247\u4e0e AQE \u51b3\u5b9a\u3002<\/p>\n<\/li>\n<hr \/>\n<h3>\u672c\u7ae0\u5c0f\u7ed3<\/h3>\n<p>\u8fd9\u4e00\u7ae0\u628a&#034;\u5c11\u8bfb\u6570\u636e&#034;\u843d\u6210\u4e86\u4e09\u6bb5\u53ef\u6267\u884c\u7684\u8bdd&#xff1a;\u5206\u533a\u5b9a\u76ee\u5f55\u3001\u5206\u6876\u5b9a\u6587\u4ef6\u3001\u6392\u5e8f\u5b9a\u5757\u3002<\/p>\n<ul>\n<li>5.1 \u7528\u8868 5-1 \u9489\u4f4f\u4e09\u79cd&#034;\u5206\u533a&#034;\u7684\u5dee\u522b&#xff0c;\u6838\u5fc3\u4e0d\u662f\u8bed\u6cd5&#xff0c;\u800c\u662f\u8c01\u8d1f\u8d23\u88c1\u526a\u2014\u2014Hive \u662f Spark&#xff0c;Iceberg \u662f\u8868\u683c\u5f0f planner&#xff0c;Paimon \u662f\u5206\u533a &#043; \u6876\u4e24\u7ea7\u5b9a\u4f4d\u3002\u56fe 5-1 \u7ed9\u51fa\u4e09\u4ef6\u4e8b\u5404\u7ba1\u54ea\u4e00\u6bb5\u7269\u7406\u5e03\u5c40\u3002<\/li>\n<li>5.2 \u7ed9\u4e86\u516d\u4e2a\u5931\u6548\u6848\u4f8b\u548c\u4e00\u5f20\u901f\u67e5\u8868\u3002\u4e24\u4e2a\u5fc5\u987b\u5e26\u8d70\u7684\u4e60\u60ef&#xff1a;\u7ed3\u8bba\u7531 files read \u7ed9&#xff0c;\u4e0d\u7531 EXPLAIN \u7ed9&#xff1b;\u683c\u5f0f\u4e0d\u4e00\u81f4\u4f1a\u8fd4\u56de\u7a7a\u800c\u4e0d\u662f\u8fd4\u56de\u6162&#xff0c;\u6240\u4ee5\u65ad\u8a00\u8981\u65e0\u6761\u4ef6\u52a0\u3002<\/li>\n<li>5.3 \u6570\u6e05\u4e86\u56db\u5c42\u7edf\u8ba1&#xff0c;\u4e5f\u6570\u6e05\u4e86 Spark \u5b98\u65b9\u771f\u6b63\u7ed9\u4e86\u5f00\u5173\u7684\u53ea\u6709\u4e09\u5c42\u2014\u2014page \u7ea7\u5728\u5b98\u65b9\u6587\u6863\u91cc\u627e\u4e0d\u5230\u4efb\u4f55\u914d\u7f6e\u9879&#xff0c;\u672c\u7ae0\u4e0d\u628a\u5b83\u5199\u6210\u4e8b\u5b9e\u3002\u9644\u5e26\u4e00\u6761\u786c\u4e8b\u5b9e&#xff1a;\u5411\u91cf\u5316\u8bfb\u53d6\u4e0e record-level filtering \u4e92\u65a5\u3002<\/li>\n<li>5.4 \u7834\u6389&#034;\u5199\u5165\u65f6\u6392\u5e8f&#034;\u7684\u8bed\u6cd5\u5e7b\u89c9&#xff0c;\u628a\u9009\u578b\u89c4\u5219\u4ea4\u7ed9\u5b98\u65b9\u533a\u95f4&#xff08;1 \u5217\u7ebf\u6027 \/ 2~4 \u5217 Z-Order \/ 5 \u5217\u4ee5\u4e0a Hilbert&#xff09;&#xff0c;\u5e76\u7ed9\u4e86\u4e00\u6b21\u771f\u5b9e\u5bf9\u7167&#xff1a;ClickHouse \u7684\u6392\u5e8f\u662f\u7d22\u5f15\u672c\u8eab&#xff0c;\u6e56\u8868\u7684\u6392\u5e8f\u53ea\u662f\u5582\u7ed9\u7edf\u8ba1\u7684\u4fe1\u606f&#xff0c;\u7cbe\u5ea6\u5355\u4f4d\u7684\u5dee\u522b\u51b3\u5b9a\u4e86\u4e0d\u8981\u62ff granule \u7684\u601d\u8def\u53bb\u5207\u6e56\u8868\u6587\u4ef6\u3002<\/li>\n<li>5.5 \u4e09\u4e2a&#034;\u6876&#034;\u4e09\u79cd\u8bed\u4e49\u3002Iceberg \u7684\u6876\u662f\u5206\u533a\u3001Paimon \u7684\u6876\u662f\u6b63\u786e\u6027\u7ea6\u675f\u3001Hive \u7684\u6876\u624d\u662f shuffle \u4f18\u5316&#xff1b;SPJ \u8ba9\u6e56\u8868\u6709\u4e86\u7406\u8bba\u4e0a\u7684\u53ef\u80fd&#xff0c;\u4f46\u90a3\u662f\u5b9e\u6d4b\u9879\u3002<\/li>\n<li>5.6 \u56db\u5c42\u6f0f\u6597\u6536\u53e3\u6210\u4e00\u5f20\u586b\u6570\u8868\u3002\u672c\u7ae0\u7684\u7ed3\u8bba\u5fc5\u987b\u7531\u8bfb\u8005\u7684\u5b9e\u6d4b\u6570\u5b57\u6765\u8865\u5b8c\u2014\u2014\u8fd9\u4e0d\u662f\u7559\u767d&#xff0c;\u8fd9\u662f\u8fd9\u4e00\u7ae0\u771f\u6b63\u6559\u7684\u4e1c\u897f&#xff1a;\u4f18\u5316\u5668\u7684\u80fd\u529b\u4e0a\u9650\u662f\u7269\u7406\u5e03\u5c40\u7ed9\u7684&#xff0c;\u800c\u5e03\u5c40\u7684\u6548\u679c\u53ea\u6709\u91cf\u8fc7\u624d\u77e5\u9053\u3002<\/li>\n<\/ul>\n<hr \/>\n<p>\u4e0b\u7ae0\u9884\u544a&#xff1a;\u5e03\u5c40\u80fd\u88c1\u6389\u6570\u636e&#xff0c;\u4f46\u524d\u63d0\u662f\u7c7b\u578b\u9009\u5bf9\u2014\u2014\u7b2c 6 \u7ae0\u300a\u6570\u636e\u7c7b\u578b\u3001VARIANT \u4e0e ANSI \u8bed\u4e49\u300b\u3002Spark 4 \u8d77 spark.sql.ansi.enabled \u9ed8\u8ba4 true \u5e26\u6765\u7684\u884c\u4e3a\u6e05\u5355&#xff0c;\u9664\u96f6\/\u6ea2\u51fa\/\u5b57\u7b26\u4e32\u8f6c\u6570\u503c\u5931\u8d25\u4ece&#034;\u9759\u9ed8\u7ed9 NULL&#034;\u53d8\u6210&#034;\u629b\u5f02\u5e38&#034;&#xff1b;Timestamp \/ TimestampNTZ \/ Date \u7684\u65f6\u533a\u8bed\u4e49&#xff08;\u56de\u6263\u524d\u4f5c&#034;Doris DATETIME \u4e0d\u5f97\u5e26\u6beb\u79d2&#034;\u4e0e&#034;Doris 4.1 Arrow DATE \u6709\u65f6\u533a\u95ee\u9898&#034;&#xff09;&#xff1b;collect_time 13 \u4f4d\u6beb\u79d2\u8981\u9664 1000 \u8fd9\u4e2a\u5751\u5728 Spark \u4fa7\u7684\u5bf9\u5e94\u5199\u6cd5&#xff1b;\u4ee5\u53ca 4.1 GA \u7684 VARIANT\u2014\u2014\u8f66\u7aef\u90a3\u79cd&#034;\u5b57\u6bb5\u6570\u91cf\u548c\u5185\u5bb9\u90fd\u4e0d\u5b9a\u7684\u4fe1\u53f7\u5305&#034;&#xff0c;\u5c55\u5f00\u6210\u5bbd\u8868\u3001\u5b58 String \u52a0 JSON \u51fd\u6570\u3001\u5b58 VARIANT&#xff0c;\u4e09\u79cd\u65b9\u6848\u7684\u5b58\u50a8\u4e0e\u67e5\u8be2\u4ee3\u4ef7\u5dee\u591a\u5c11\u3002<\/p>\n<hr \/>\n<h3>\u5b98\u65b9\u53c2\u8003\u8d44\u6599<\/h3>\n<table>\n<tr>\u4e3b\u9898\u94fe\u63a5<\/tr>\n<tbody>\n<tr>\n<td>Spark CREATE TABLE&#xff08;Hive \u683c\u5f0f&#xff0c;CLUSTERED BY\/SORTED BY\/INTO n BUCKETS \u539f\u6587\u4e0e NOTE&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/sql-ref-syntax-ddl-create-table-hiveformat.html<\/td>\n<\/tr>\n<tr>\n<td>Spark CREATE TABLE&#xff08;Data Source \u683c\u5f0f&#xff0c;\u542b USING CSV &#8230; CLUSTERED BY (Id) INTO 4 buckets \u793a\u4f8b&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/sql-ref-syntax-ddl-create-table-datasource.html<\/td>\n<\/tr>\n<tr>\n<td>Spark SQL Performance Tuning&#xff08;Storage Partition Join\u3001filterPushdown\u3001files \u5206\u7247\u53c2\u6570&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/sql-performance-tuning.html<\/td>\n<\/tr>\n<tr>\n<td>Spark Configuration&#xff08;metastorePartitionPruning\u3001DPP\u3001runtime bloom filter\u3001sources.bucketing.*&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/configuration.html<\/td>\n<\/tr>\n<tr>\n<td>Spark Parquet \u6570\u636e\u6e90\u9875&#xff08;\u5411\u91cf\u5316\u8bfb\u53d6\u3001record-level filtering&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/sql-data-sources-parquet.html<\/td>\n<\/tr>\n<tr>\n<td>Spark DataSource V2&#xff08;RequiresDistributionAndOrdering\u3001requested distribution &amp; ordering&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/sql-data-sources-v2.html<\/td>\n<\/tr>\n<tr>\n<td>Spark SELECT \u8bed\u6cd5&#xff08;ORDER BY \/ CLUSTER BY \u7684\u5b98\u65b9\u5b9a\u4f4d\u4e0e\u4e92\u65a5\u5173\u7cfb&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/docs\/latest\/sql-ref-syntax-qry-select.html<\/td>\n<\/tr>\n<tr>\n<td>Spark 4.1.0 Release Notes&#xff08;SPARK-53074 SPJ\u3001SPARK-51831 DSv2 \u5217\u88c1\u526a\u3001SPARK-52187 join pushdown&#xff09;<\/td>\n<td>https:\/\/spark.apache.org\/releases\/spark-release-4.1.0.html<\/td>\n<\/tr>\n<tr>\n<td>Iceberg Spark DDL&#xff08;WRITE ORDERED BY \/ WRITE LOCALLY ORDERED BY \/ WRITE UNORDERED\u3001partition transforms \u5217\u8868&#xff09;<\/td>\n<td>https:\/\/iceberg.apache.org\/docs\/latest\/spark-ddl\/<\/td>\n<\/tr>\n<tr>\n<td>Iceberg Spark \u5199\u5165\u4e0e\u914d\u7f6e&#xff08;spark.sql.iceberg.distribution-mode\u3001write.distribution-mode&#xff09;<\/td>\n<td>https:\/\/iceberg.apache.org\/docs\/latest\/spark-writes\/<\/td>\n<\/tr>\n<tr>\n<td>Iceberg \u8868\u5c5e\u6027&#xff08;write.target-file-size-bytes\u3001row-group\/page \u5927\u5c0f\u3001write.metadata.metrics.default&#xff09;<\/td>\n<td>https:\/\/iceberg.apache.org\/docs\/latest\/configuration\/<\/td>\n<\/tr>\n<tr>\n<td>Iceberg \u5b58\u50a8\u8fc7\u7a0b&#xff08;rewrite_data_files \u7684 strategy\/sort_order\/where\/options \u4e0e\u9ed8\u8ba4\u503c&#xff09;<\/td>\n<td>https:\/\/iceberg.apache.org\/docs\/latest\/spark-procedures\/<\/td>\n<\/tr>\n<tr>\n<td>Iceberg \u89c4\u8303&#xff1a;partition transforms \u4e0e\u6587\u4ef6\u7edf\u8ba1\u5b57\u6bb5&#xff08;108~128&#xff09;<\/td>\n<td>https:\/\/iceberg.apache.org\/spec\/<\/td>\n<\/tr>\n<tr>\n<td>Iceberg \u9690\u85cf\u5206\u533a\u4e0e\u5206\u533a\u6f14\u8fdb<\/td>\n<td>https:\/\/iceberg.apache.org\/docs\/latest\/partitioning\/ \u4e0e \u2026\/evolution\/<\/td>\n<\/tr>\n<tr>\n<td>Iceberg \u6027\u80fd\u9875&#xff1a;data filtering&#xff08;planner \u4e24\u6b65\u88c1\u526a&#xff09;<\/td>\n<td>https:\/\/iceberg.apache.org\/docs\/latest\/performance\/<\/td>\n<\/tr>\n<tr>\n<td>Paimon \u6570\u636e\u5206\u5e03&#xff08;\u56fa\u5b9a\/\u52a8\u6001\/postpone \u4e09\u6876\u6a21\u5f0f\u3001bucket sizing \u5b98\u65b9\u53e3\u5f84&#xff09;<\/td>\n<td>https:\/\/paimon.apache.org\/docs\/master\/primary-key-table\/data-distribution\/<\/td>\n<\/tr>\n<tr>\n<td>Paimon \u589e\u91cf\u805a\u7c07&#xff08;clustering.strategy \u7684 auto\/order\/zorder\/hilbert \u89c4\u5219&#xff09;<\/td>\n<td>https:\/\/paimon.apache.org\/docs\/master\/append-table\/incremental-clustering\/<\/td>\n<\/tr>\n<tr>\n<td>Paimon \u4e13\u5c5e\u538b\u7f29\u4e0e Sort Compact<\/td>\n<td>https:\/\/paimon.apache.org\/docs\/master\/maintenance\/dedicated-compaction\/<\/td>\n<\/tr>\n<tr>\n<td>Paimon \u5206\u533a\u7ba1\u7406&#xff08;partition.expiration-time \u7b49&#xff09;<\/td>\n<td>https:\/\/paimon.apache.org\/docs\/master\/maintenance\/manage-partitions\/<\/td>\n<\/tr>\n<tr>\n<td>\u524d\u4f5c\u5bf9\u9f50\u70b9&#xff1a;ClickHouse \u4e3b\u952e\u3001\u7a00\u758f\u7d22\u5f15\u4e0e\u67e5\u8be2\u88c1\u526a&#xff08;granule\u3001EXPLAIN indexes&#xff09;<\/td>\n<td>\u672c\u4ed3\u5e93 tech\/ck\/\u7b2c4\u7ae0 \u4e3b\u952e\u3001\u7a00\u758f\u7d22\u5f15\u4e0e\u67e5\u8be2\u88c1\u526a.md<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>\u7248\u672c\u53d6\u8bc1\u8bf4\u660e<\/h3>\n<p>\u672c\u7ae0\u53d6\u8bc1\u65e5\u671f&#xff1a;2026-11-03\u3002\u6293\u53d6\u8def\u5f84\u8bf4\u660e&#xff1a;https:\/\/spark.apache.org\/docs\/latest\/* \u5f53\u524d\u6e32\u67d3\u4e3a 4.3.0 \u6587\u6863&#xff0c;\u56e0\u6b64\u6d89\u53ca Spark \u4fa7\u7684\u6bcf\u4e00\u6761\u90fd\u53e6\u884c\u6bd4\u5bf9\u4e86 4.1.4 \u540c\u540d\u9875\u9762&#xff1b;Iceberg \u6309 docs\/latest&#xff08;\u9762\u5305\u5c51\u6807\u6ce8 Latest &#061; 1.12.0&#xff09;\u4e0e\u4ed3\u5e93 docs\/docs\/*.md \u6e90\u6587\u4ef6\u53d6\u8bc1&#xff1b;Paimon \u5b98\u65b9\u7ad9\u70b9\u4e3a\u5ba2\u6237\u7aef\u6e32\u67d3\u3001\u6293\u53d6\u4e0d\u7a33\u5b9a&#xff0c;\u9010\u5b57\u539f\u6587\u53d6\u81ea Apache Paimon \u4ed3\u5e93 master \u5206\u652f\u6587\u6863\u6e90&#xff0c;1.4.2 \u7684\u63aa\u8f9e\u672a\u9010\u5b57\u9a8c\u8bc1&#xff0c;\u51e1\u5f15\u7528\u5904\u5747\u6807\u6ce8\u4e3a master\u3002<\/p>\n<table>\n<tr>\u5185\u5bb9\u6863\u4f4d\u8bf4\u660e<\/tr>\n<tbody>\n<tr>\n<td>spark.sql.hive.metastorePartitionPruning \u9ed8\u8ba4 true&#xff0c;\u201csome predicates will be pushed down into the Hive metastore so that unmatching partitions can be eliminated earlier\u201d&#xff08;1.5.0 \u8d77&#xff09;<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>configuration \u9875\u3002\u6ce8\u610f&#xff1a;\u7ec6\u5316\u5927\u7eb2\u91cc\u9884\u5199\u7684 spark.sql.hive.metastore.partition.prune.enabled \u8fd9\u4e2a\u914d\u7f6e\u540d\u5728\u5b98\u65b9\u6587\u6863\u4e2d\u4e0d\u5b58\u5728&#xff0c;\u6b63\u6587\u5df2\u6309\u5b98\u65b9\u540d\u6539\u5199<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.hive.metastorePartitionPruningFastFallback \u9ed8\u8ba4 false&#xff08;3.3.0 \u8d77&#xff09;&#xff1b;\u5f00\u542f\u540e\u6539\u4e3a&#034;\u5148\u53d6\u5206\u533a\u540d\u518d\u5728\u5ba2\u6237\u7aef\u7b97\u8fc7\u6ee4\u8868\u8fbe\u5f0f&#034;&#xff0c;\u542b TimeZoneAwareExpression \u7684\u8c13\u8bcd\u4e0d\u652f\u6301<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>configuration \u9875<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.optimizer.dynamicPartitionPruning.enabled \u9ed8\u8ba4 true&#xff08;3.0.0 \u8d77&#xff09;&#xff0c;\u201cwe will generate predicate for partition column when it\u2019s used as join key\u201d<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>configuration \u9875\u3002\u6ce8\u610f&#xff1a;performance-tuning \u9875&#xff08;4.3.0 \u4e0e 4.1.4&#xff09;\u6b63\u6587\u6ca1\u6709&#034;Dynamic Partition Pruning&#034;\u5c0f\u8282&#xff0c;\u53ea\u6709 configuration \u9875\u6709\u8be5\u914d\u7f6e<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.optimizer.runtime.bloomFilter.enabled \u9ed8\u8ba4 true&#xff08;3.3.0 \u8d77&#xff09;&#xff0c;\u63aa\u8f9e\u4e3a&#034;insert a bloom filter in the other side to reduce shuffle data&#034;&#xff1b;numBits&#061;8388608\u3001expectedNumItems&#061;1000000\u3001maxNumBits&#061;67108864\u3001maxNumItems&#061;4000000\u3001creationSideThreshold&#061;10MB\u3001applicationSideScanSizeThreshold&#061;10GB<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>configuration \u9875\u3002\u65e7\u5e16\u5e38\u5199\u7684 applicationFrequencyThreshold \/ numberOfBits \/ numberOfApplications \u5728\u5f53\u524d\u9875\u9762\u5747\u65e0\u5339\u914d<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.parquet.filterPushdown \u9ed8\u8ba4 true&#xff08;1.2.0 \u8d77&#xff09;&#xff1b;\u5b98\u65b9\u63cf\u8ff0\u4ec5\u4e00\u53e5&#xff0c;\u4e0d\u5b58\u5728&#034;\u4ec5\u5f15\u7528\u5355\u5217\u7684\u8c13\u8bcd\u624d\u4e0b\u63a8&#034;\u7684\u63aa\u8f9e<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>parquet \u9875 &#043; configuration \u9875\u4e24\u5904\u4e00\u81f4<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.parquet.recordLevelFilter.enabled \u9ed8\u8ba4 false&#xff08;2.3.0 \u8d77&#xff09;&#xff1b;\u539f\u6587\u542b&#034;only has an effect when filterPushdown is enabled and the vectorized reader is not used&#034;<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>\u672c\u7ae0 5.3.2 \u7684\u7ed3\u8bba\u76f4\u63a5\u7531\u6b64\u63a8\u51fa\u3002\u793e\u533a\u6587\u7ae0\u5e38\u7528\u7684 spark.sql.parquet.recordColumnFilter \u4e0d\u662f\u5b98\u65b9\u914d\u7f6e\u540d<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.parquet.enableVectorizedReader \/ spark.sql.orc.enableVectorizedReader \u9ed8\u8ba4 true&#xff1b;spark.sql.orc.filterPushdown \u9ed8\u8ba4 true<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>parquet \/ configuration \u9875<\/td>\n<\/tr>\n<tr>\n<td>Parquet page index \/ column index&#xff1a;Spark \u4fa7\u914d\u7f6e\u9879<\/td>\n<td>\u5b98\u65b9\u65e0\u6b64\u5f00\u5173<\/td>\n<td>parquet\u3001configuration\u3001performance-tuning \u4e09\u9875\u5168\u6587\u65e0 page index \/ column index \/ page-level \u5b57\u6837&#xff0c;\u4e5f\u65e0 columnIndexPageSize&#xff1b;\u6b63\u6587\u6309&#034;\u6ca1\u6709\u5b98\u65b9\u4f9d\u636e&#034;\u5904\u7406&#xff0c;\u672a\u5199\u6210\u4e8b\u5b9e<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.files.maxPartitionBytes \u9ed8\u8ba4 134217728&#xff08;128 MB&#xff09;&#xff1b;performance \u9875\u5199 134217728 (128 MB)\u3001configuration \u9875\u5199 128MB&#xff0c;\u4e24\u5904\u8868\u8ff0\u4e0d\u540c<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>\u540c\u9875\u8fd8\u6ce8\u660e&#034;effective only when using file-based sources such as Parquet, JSON and ORC&#034;<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.files.openCostInBytes \u9ed8\u8ba4 4194304&#xff08;4 MB&#xff09;&#xff1b;ignoreCorruptFiles \u9ed8\u8ba4 false<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>performance-tuning \u9875<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.sources.bucketing.enabled \u9ed8\u8ba4 true&#xff08;2.0.0 \u8d77&#xff0c;\u201cWhen false, we will treat bucketed table as normal table\u201d&#xff09;&#xff1b;maxBuckets \u9ed8\u8ba4 100000&#xff08;2.4.0 \u8d77&#xff09;<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>configuration \u9875<\/td>\n<\/tr>\n<tr>\n<td>SPJ \u5b9a\u4f4d\u539f\u6587&#xff08;\u201cgeneralization of the concept of Bucket Joins \u2026 compatible V2 DataSources\u201d&#xff09;\u4e0e spark.sql.sources.v2.bucketing.enabled \u9ed8\u8ba4 true&#xff08;3.3.0 \u8d77&#xff09;<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>performance-tuning \u9875 &#043; configuration \u9875<\/td>\n<\/tr>\n<tr>\n<td>\u5b98\u65b9 INSERT \u8bed\u6cd5\u9875\u4e0d\u542b ORDER BY \/ SORT BY \/ CLUSTER BY \/ DISTRIBUTE BY \u5b50\u53e5<\/td>\n<td>\u5b98\u65b9\u539f\u6587&#xff08;\u8d1f\u9762\u7ed3\u8bba&#xff09;<\/td>\n<td>\u5168\u9875 0 \u6b21\u5339\u914d&#xff0c;5.4.1 \u7684\u7ed3\u8bba\u7531\u6b64\u6765<\/td>\n<\/tr>\n<tr>\n<td>SELECT \u7684 ORDER BY \u5b9a\u4f4d\u539f\u6587\u3001\u4e0e SORT\/CLUSTER\/DISTRIBUTE \u4e92\u65a5&#xff1b;CLUSTER BY \u201cdoes not guarantee a total order of output\u201d<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>sql-ref-syntax-qry-select \u7cfb\u5217\u9875<\/td>\n<\/tr>\n<tr>\n<td>DSv2 RequiresDistributionAndOrdering \u539f\u6587&#xff0c;\u542b&#034;enabling optimized data layouts such as clustering or Z-ordering&#034;<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>sql-data-sources-v2 \u9875<\/td>\n<\/tr>\n<tr>\n<td>Spark 4.1.0 \u4e2d\u4e0e\u672c\u7ae0\u76f8\u5173\u7684\u6761\u76ee&#xff1a;SPARK-53074&#xff08;SPJ partial clustering&#xff09;\u3001SPARK-51831&#xff08;DSv2 existsJoin \u5217\u88c1\u526a&#xff09;\u3001SPARK-52187&#xff08;DSv2 join pushdown&#xff09;<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>4.1.0 \u53d1\u5e03\u8bf4\u660e\u3002\u8be5\u9875\u65e0 bucketing \/ sorted write \/ data skipping \u7c7b\u6761\u76ee&#xff0c;\u6b63\u6587\u672a\u865a\u6784 4.1 \u76f8\u5173\u65b0\u7279\u6027<\/td>\n<\/tr>\n<tr>\n<td>spark.sql.optimizer.canChangeCachedPlanOutputPartitioning\u3001dynamicPartitionPruning.reuseBroadcastOnly<\/td>\n<td>\u4e0d\u5b58\u5728\u4e8e\u5f53\u524d\u5b98\u65b9\u9875<\/td>\n<td>4.3.0 \u4e0e 4.1.4 \u5168\u6587\u65e0\u5339\u914d&#xff0c;\u6b63\u6587\u672a\u5f15\u7528<\/td>\n<\/tr>\n<tr>\n<td>Iceberg ALTER TABLE &#8230; WRITE ORDERED BY \/ WRITE LOCALLY ORDERED BY \/ WRITE UNORDERED \/ WRITE DISTRIBUTED BY PARTITION&#xff1b;\u201cMERGE INTO in Spark will use the table ordering\u201d&#xff1b;\u6269\u5c55\u6a21\u5757\u81ea 0.11.0<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>spark-ddl \u9875\u3002\u6ce8\u610f&#xff1a;CREATE TABLE &#8230; WRITE ORDER BY \u8fd9\u79cd\u5199\u6cd5\u5728\u5b98\u65b9\u6587\u6863\u4e2d\u4e0d\u5b58\u5728&#xff0c;\u672c\u7ae0\u4e00\u5f8b\u7528 ALTER \u5f62\u5f0f<\/td>\n<\/tr>\n<tr>\n<td>Iceberg partition transforms \u6e05\u5355&#xff08;year\/month\/day(&#061;dateint)\/hour\/bucket(N,col)\/truncate(L,col)&#xff09;\u4e0e spec \u4e2d bucket[N]&#061;\u201cHash of value, mod N\u201d<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>spark-ddl &#043; spec \u9875&#xff1b;\u65e7\u5f0f\u590d\u6570\u5199\u6cd5 days() \u7b49\u5b98\u65b9\u6ce8\u660e&#034;also supported for compatibility&#034;<\/td>\n<\/tr>\n<tr>\n<td>Iceberg rewrite_data_files \u53c2\u6570\u96c6&#xff08;strategy \u9ed8\u8ba4 binpack\u3001sort_order \u652f\u6301 zorder(&#8230;)\/hilbert(&#8230;)\u3001where\u3001options&#xff09;\u4e0e\u5168\u90e8\u901a\u7528\u9009\u9879\u9ed8\u8ba4\u503c<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>spark-procedures \u9875\u3002\u65e7\u53c2\u6570\u540d sort_by\/zorder_by\/distribute_by\/sort_buffer_size \u5728\u8be5\u9875\u51fa\u73b0\u6b21\u6570\u4e3a 0&#xff1b;sort_by \u4ec5\u5b58\u5728\u4e8e rewrite_manifests<\/td>\n<\/tr>\n<tr>\n<td>Iceberg write.target-file-size-bytes \u9ed8\u8ba4 512MB\u3001write.parquet.row-group-size-bytes \u9ed8\u8ba4 128MB\u3001write.parquet.page-size-bytes \u9ed8\u8ba4 1MB\u3001write.parquet.dict-size-bytes \u9ed8\u8ba4 2MB\u3001write.metadata.delete-after-commit.enabled \u9ed8\u8ba4 false<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>configuration \u9875\u9010\u9879<\/td>\n<\/tr>\n<tr>\n<td>Iceberg write.metadata.metrics.default \u9ed8\u8ba4 truncate(16)&#xff0c;\u53ef\u9009 none\/counts\/truncate(length)\/full<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>configuration \u9875\u3002\u6309\u5217\u8986\u76d6\u7684\u914d\u7f6e\u540d\u672c\u6b21\u672a\u9010\u5b57\u53d6\u8bc1&#xff0c;\u6b63\u6587\u5199\u4e3a&#034;\u53ef\u6309\u5217\u8986\u76d6&#xff0c;\u540d\u79f0\u9700\u6309\u5b98\u65b9\u9875\u6838\u5bf9&#034;<\/td>\n<\/tr>\n<tr>\n<td>Iceberg planner \u4e24\u6b65\u88c1\u526a\u539f\u6587&#xff08;\u5148 partition \u8c13\u8bcd\u3001\u518d value counts\/null counts\/lower\/upper bounds \u6d88\u6587\u4ef6&#xff09;\u4e0e spec \u5b57\u6bb5 108\/109\/110\/137\/125\/128<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>performance \u9875 &#043; spec \u9875<\/td>\n<\/tr>\n<tr>\n<td>Iceberg \u201cPartition evolution is a metadata operation and does not eagerly rewrite files\u201d&#xff1b;\u8001\u6570\u636e\u4fdd\u7559\u65e7 spec&#xff08;split planning&#xff09;<\/td>\n<td>\u5b98\u65b9\u539f\u6587<\/td>\n<td>evolution \u9875<\/td>\n<\/tr>\n<tr>\n<td>&#034;\u5206\u533a\u6570\u91cf\u63a7\u5236\u5728\u51e0\u5343\u91cf\u7ea7&#034;\u8fd9\u4e00\u63aa\u8f9e<\/td>\n<td>\u5b98\u65b9\u672a\u627e\u5230<\/td>\n<td>Iceberg partitioning \u6587\u6863\u5168\u6587\u5bf9 thousand \u5339\u914d\u6570\u4e3a 0\u3002\u6b63\u6587\u6807\u6ce8\u4e3a\u524d\u4f5c\u5de5\u7a0b\u53e3\u5f84&#xff0c;\u4e0d\u4f5c\u4e3a\u5b98\u65b9\u89c4\u8303<\/td>\n<\/tr>\n<tr>\n<td>Paimon \u4e09\u6876\u6a21\u5f0f\u8868&#xff08;\u56fa\u5b9a \/ \u52a8\u6001&#061;\u9ed8\u8ba4 -1 \/ postpone -2&#xff09;\u4e0e\u5404\u81ea planning \u7ea6\u675f\u539f\u6587&#xff1b;Math.abs(key_hashcode % numBuckets)&#xff1b;bucket-key \u9ed8\u8ba4\u53d6&#034;\u4e3b\u952e\u53bb\u6389\u5206\u533a\u5217&#034;<\/td>\n<td>\u5b98\u65b9\u539f\u6587&#xff08;master&#xff09;<\/td>\n<td>data-distribution &#043; \u751f\u6210\u7684 core_configuration\u30021.4.2 \u63aa\u8f9e\u672a\u9010\u5b57\u9a8c\u8bc1<\/td>\n<\/tr>\n<tr>\n<td>Paimon bucket sizing&#xff1a;\u201cFor MOR workloads, 200 MB\u20131 GB per bucket is an initial sizing guide, not a limit\u201d&#xff1b;\u201cbucket count is not a hard read-parallelism cap\u201d<\/td>\n<td>\u5b98\u65b9\u539f\u6587&#xff08;master&#xff09;<\/td>\n<td>data-distribution#bucket-sizing\u3002\u5927\u7eb2\u539f\u8bb0\u7684&#034;\u5355\u6876 200MB \u5de6\u53f3&#034;\u4e0d\u591f\u51c6\u786e&#xff0c;\u5df2\u6309\u5b98\u65b9\u5b8c\u6574\u63aa\u8f9e\u6539\u5199\u5e76\u8865\u4e0a\u540e\u534a\u53e5<\/td>\n<\/tr>\n<tr>\n<td>Paimon \u5206\u533a\u8fc7\u671f&#xff1a;partition.expiration-time \u9ed8\u8ba4 (none)\u3001expiration-strategy&#061;values-time\u3001check-interval&#061;1h\u3001expiration-retain-latest&#061;false\u3001dynamic-bucket.target-row-num&#061;2000000<\/td>\n<td>\u5b98\u65b9\u539f\u6587&#xff08;master&#xff09;<\/td>\n<td>manage-partitions \u9875&#xff0c;\u7b2c 9\/10 \u7ae0\u7ee7\u7eed\u5f15\u7528<\/td>\n<\/tr>\n<tr>\n<td>Paimon \u652f\u6301\u6392\u5e8f\u4e0e Z-Order&#xff1a;clustering.strategy \u53ef\u9009 auto\/order\/zorder\/hilbert&#xff0c;auto \u89c4\u5219\u4e3a&#034;1 \u5217 order\u30012~4 \u5217 zorder\u30015 \u5217\u4ee5\u4e0a hilbert&#034;&#xff1b;Sort Compact CALL sys.compact(order_strategy &#061;&gt; &#039;zorder&#039;, order_by &#061;&gt; &#039;a,b&#039;)&#xff0c;\u5b98\u65b9\u9650\u5b9a&#034;currently supports only bucket&#061;-1 and batch mode&#034;<\/td>\n<td>\u5b98\u65b9\u539f\u6587&#xff08;master&#xff09;<\/td>\n<td>incremental-clustering &#043; dedicated-compaction \u9875\u3002\u53d6\u8bc1\u524d\u6b63\u6587\u66fe\u51c6\u5907\u5199&#034;Paimon \u65e0\u6392\u5e8f\u80fd\u529b&#034;&#xff0c;\u8be5\u7ed3\u8bba\u5df2\u88ab\u63a8\u7ffb\u5e76\u6539\u5199<\/td>\n<\/tr>\n<tr>\n<td>Paimon Spark \u4fa7 ORDER BY \/ DISTRIBUTE BY \/ CLUSTER BY \u5199\u5165\u652f\u6301<\/td>\n<td>\u5b98\u65b9\u6587\u6863\u672a\u627e\u5230<\/td>\n<td>spark\/sql-write \u4e0e sql-query \u4e24\u9875\u5bf9\u8fd9\u4e9b\u5173\u952e\u5b57\u5339\u914d\u6570\u4e3a 0&#xff1b;\u6b63\u6587\u672a\u7ed9\u51fa\u4efb\u4f55 Paimon \u4fa7\u7684\u5199\u5165\u6392\u5e8f\u8bed\u6cd5<\/td>\n<\/tr>\n<tr>\n<td>Paimon Spark \u9ed8\u8ba4 static \u8986\u76d6&#xff0c;\u9700 spark.sql.sources.partitionOverwriteMode&#061;dynamic<\/td>\n<td>\u5b98\u65b9\u539f\u6587&#xff08;master&#xff09;<\/td>\n<td>sql-write \u9875&#xff0c;\u7b2c 9 \u7ae0\u7ee7\u7eed\u7528<\/td>\n<\/tr>\n<tr>\n<td>\u5411\u91cf\u5316 reader \u4e0d\u652f\u6301\u7684\u7c7b\u578b\u6e05\u5355\u3001\u6279\u5927\u5c0f\u914d\u7f6e\u540d<\/td>\n<td>\u672a\u9010\u5b57\u53d6\u8bc1<\/td>\n<td>\u6b63\u6587\u53ea\u7ed9&#034;\u770b Batched \u5224\u65ad\u8def\u5f84&#034;\u7684\u5b9e\u6d4b\u65b9\u6cd5&#xff0c;\u672a\u5217\u5177\u4f53\u7c7b\u578b\u4e0e\u6570\u503c<\/td>\n<\/tr>\n<tr>\n<td>Spark \u5bf9 Iceberg\/Paimon \u7684 SPJ \u5b9e\u9645\u751f\u6548\u6027<\/td>\n<td>\u5f85\u5b9e\u6d4b<\/td>\n<td>\u5b98\u65b9\u53ea\u627f\u8bfa&#034;compatible V2 DataSources&#034;&#xff1b;\u6b63\u6587\u4e00\u5f8b\u8981\u6c42&#034;\u6570 Exchange&#034;\u81ea\u8bc1<\/td>\n<\/tr>\n<tr>\n<td>EXPLAIN \u8f93\u51fa\u793a\u4f8b\u4e2d\u7684 files read\u3001row groups read: Selected\/Missing\/Total \u7b49\u6307\u6807\u540d\u4e0e\u4e8b\u4ef6\u65e5\u5fd7\u5b57\u6bb5\u62fc\u5199<\/td>\n<td>\u7b14\u8005\u793a\u610f &#043; \u5b9e\u6d4b\u9879<\/td>\n<td>\u5f62\u6001\u53d6\u81ea\u771f\u5b9e UI\/\u4e8b\u4ef6\u65e5\u5fd7&#xff0c;\u540d\u79f0\u968f\u7248\u672c\u5fae\u8c03&#xff1b;measure_scan.py \u7684\u7b56\u7565\u662f&#034;\u5148\u5168\u6253\u51fa\u6765\u518d\u8ba4&#034;<\/td>\n<\/tr>\n<tr>\n<td>\u8868 5-5\u3001\u56fe 5-2 \u7684\u5c42\u7ea7\u987a\u5e8f\u30015.4.4 \u7684&#034;\u6392\u5e8f\u662f\u7d22\u5f15 vs \u6392\u5e8f\u5582\u7edf\u8ba1&#034;\u7ed3\u8bba\u30015.6.3 \u4e09\u6761\u8bfb\u8868\u7ed3\u8bba<\/td>\n<td>\u7b14\u8005\u5f52\u7eb3<\/td>\n<td>\u5404\u5c42\u673a\u5236\u672c\u8eab\u6709\u5b98\u65b9\u539f\u6587&#xff0c;\u5206\u5c42\u4e0e\u6027\u4ef7\u6bd4\u7ed3\u8bba\u662f\u672c\u4e66\u7ec4\u7ec7\u65b9\u5f0f&#xff0c;\u975e\u5b98\u65b9\u5206\u7c7b<\/td>\n<\/tr>\n<tr>\n<td>\u56db\u6001\u5b9e\u9a8c&#xff08;S0~S3&#xff09;\u7684\u5b9e\u6d4b\u6570\u5b57<\/td>\n<td>\u672c\u7cfb\u5217\u4e0d\u4ee3\u586b<\/td>\n<td>\u6587\u4ef6\u6570\u4e0e\u5757\u6570\u5f3a\u4f9d\u8d56\u7248\u672c\/\u8fde\u63a5\u5668&#xff1b;\u586b\u6570\u8868\u8981\u6c42\u8bfb\u8005\u5728\u81ea\u8eab\u73af\u5883\u91cf\u51fa\u6765<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>\u5b9e\u64cd\u6e05\u5355<\/h3>\n<p>\u524d\u7f6e&#xff1a;\u7b2c 2 \u7ae0\u73af\u5883&#xff08;4.1.4 \u4e3b\u7ebf&#xff0c;Paimon \u76f8\u5173\u6b65\u9aa4\u5982\u9047\u4e0d\u517c\u5bb9\u6539\u5728 4.0.5 \u8dd1&#xff09;\u3001paimon.demo.ods_vehicle_telemetry 20 \u4e07\u884c\u3001iceberg.demo.dws_vehicle_daily \u4e0e iceberg.demo.dim_vehicle_info\u3001\u4e8b\u4ef6\u65e5\u5fd7\u5df2\u5f00\u3002\u672c\u7ae0\u65b0\u5efa iceberg.demo.dwd_vehicle_telemetry \u4e0e\u56db\u5f20 t_layout_*\u3002<\/p>\n<table>\n<tr>#\u64cd\u4f5c\u547d\u4ee4 \/ \u4f4d\u7f6e\u9a8c\u6536\u6807\u51c6<\/tr>\n<tbody>\n<tr>\n<td>1<\/td>\n<td>\u770b\u6e05\u5206\u533a\u503c\u5f62\u6001<\/td>\n<td>SHOW PARTITIONS paimon.demo.ods_vehicle_telemetry<\/td>\n<td>30 \u884c\u3001\u503c\u5168\u4e3a yyyy-MM-dd&#xff1b;\u4e0d\u7b26\u5c31\u5148\u4fee\u6570\u636e&#xff0c;\u522b\u5f80\u4e0b\u505a<\/td>\n<\/tr>\n<tr>\n<td>2<\/td>\n<td>\u590d\u73b0\u5931\u6548\u6848\u4f8b 1<\/td>\n<td>to_date(__dt__)&#061;&#039;2026-10-15&#039; &#043; EXPLAIN FORMATTED<\/td>\n<td>PartitionFilters: []&#xff1b;\u7528 _hist_ \u6307\u51fa\u8c13\u8bcd\u5728\u54ea\u4e00\u6b65\u88ab\u5224\u4e0d\u53ef\u4e0b\u63a8<\/td>\n<\/tr>\n<tr>\n<td>3<\/td>\n<td>\u590d\u73b0\u5931\u6548\u6848\u4f8b 2<\/td>\n<td>__dt__ &#061; 20261015<\/td>\n<td>\u8bb0\u5f55\u662f&#034;\u62a5\u9519&#034;\u8fd8\u662f&#034;0 \u884c&#034;&#xff0c;\u5e76\u628a\u7ed3\u8bba\u5199\u8fdb 5.2.6 \u5dee\u5f02\u8868<\/td>\n<\/tr>\n<tr>\n<td>4<\/td>\n<td>\u590d\u73b0\u5931\u6548\u6848\u4f8b 3<\/td>\n<td>OR \u7248\u4e0e AND \u7248\u5404\u8dd1\u4e00\u6b21<\/td>\n<td>OR \u7248 files read \u2248 30 \u5206\u533a&#xff1b;AND \u7248 \u2248 1 \u5206\u533a<\/td>\n<\/tr>\n<tr>\n<td>5<\/td>\n<td>\u590d\u73b0\u5931\u6548\u6848\u4f8b 4<\/td>\n<td>&lt;&gt; \/ &gt; \/ IN \u4e09\u79cd\u5199\u6cd5<\/td>\n<td>\u4e09\u8005 PartitionFilters \u90fd\u975e\u7a7a&#xff0c;\u4f46 files read \u5206\u522b \u224829 \/ 7 \/ 7<\/td>\n<\/tr>\n<tr>\n<td>6<\/td>\n<td>\u590d\u73b0\u5931\u6548\u6848\u4f8b 5<\/td>\n<td>\u7d27\u51d1\u683c\u5f0f\u67e5\u8be2 &#043; raise_error \u65ad\u8a00<\/td>\n<td>\u524d\u8005\u8fd4\u56de 0 \u4e0d\u62a5\u9519&#xff1b;\u540e\u8005\u6539\u6210\u4e0d\u5b58\u5728\u7684\u5206\u533a\u503c\u65f6\u5fc5\u987b\u8ba9\u4f5c\u4e1a\u5931\u8d25<\/td>\n<\/tr>\n<tr>\n<td>7<\/td>\n<td>\u590d\u73b0\u5931\u6548\u6848\u4f8b 6<\/td>\n<td>CTE \u6d3e\u751f\u5217\u7248 vs \u8c13\u8bcd\u4e0b\u63a8\u7248<\/td>\n<td>\u6d3e\u751f\u5217\u7248 PartitionFilters \u4e3a\u7a7a&#xff1b;\u4e24\u7248\u7ed3\u679c\u884c\u6570\u4e00\u81f4<\/td>\n<\/tr>\n<tr>\n<td>8<\/td>\n<td>\u89c2\u5bdf\u88c1\u526a\u53d1\u751f\u5728\u54ea\u4e00\u5c42<\/td>\n<td>\u540c\u4e00\u6761\u4ef6\u5206\u522b\u6253\u5230 Paimon \/ Iceberg \/ Hive \u8868<\/td>\n<td>\u80fd\u8bf4\u51fa\u4e09\u8005\u5019\u9009\u6587\u4ef6\u6311\u9009\u5206\u522b\u7531\u8c01\u5b8c\u6210<\/td>\n<\/tr>\n<tr>\n<td>9<\/td>\n<td>\u786e\u8ba4\u8bfb\u53d6\u8def\u5f84<\/td>\n<td>EXPLAIN FORMATTED \u770b Batched<\/td>\n<td>\u80fd\u5224\u65ad\u5f53\u524d\u8d70\u7684\u662f\u5411\u91cf\u5316\u8fd8\u662f\u975e\u5411\u91cf\u5316\u8def\u5f84<\/td>\n<\/tr>\n<tr>\n<td>10<\/td>\n<td>\u9a8c\u8bc1 record-level filter \u53d6\u820d<\/td>\n<td>enableVectorizedReader&#061;false &#043; recordLevelFilter.enabled&#061;true<\/td>\n<td>\u5bf9\u6bd4 scanned rows \u4e0e\u8017\u65f6&#xff1b;\u8bb0\u5f55&#034;\u662f\u5426\u503c\u5f97\u6362\u8def\u5f84&#034;\u7684\u7ed3\u8bba<\/td>\n<\/tr>\n<tr>\n<td>11<\/td>\n<td>\u5efa DWD \u8868\u5e76\u8bbe\u6392\u5e8f<\/td>\n<td>5.4.3 \u6b65\u9aa4\u2460\u2461\u2464<\/td>\n<td>rows_total&#061;200000\u3001.files \u91cc\u6587\u4ef6\u6570 \u22652<\/td>\n<\/tr>\n<tr>\n<td>12<\/td>\n<td>Z-Order \u91cd\u5199\u5386\u53f2\u6570\u636e<\/td>\n<td>5.4.3 \u6b65\u9aa4\u2465<\/td>\n<td>\u91cd\u5199\u524d\u540e sum(record_count) \u5b8c\u5168\u76f8\u7b49&#xff1b;file_cnt \u7b26\u5408 target \u8bbe\u5b9a<\/td>\n<\/tr>\n<tr>\n<td>13<\/td>\n<td>\u9a8c\u8bc1&#034;\u6392\u5e8f\u53ea\u7ba1\u65b0\u5199\u5165&#034;<\/td>\n<td>\u53ea\u6267\u884c\u2461\u4e0d\u6267\u884c\u2465&#xff0c;\u8dd1 Q2 \u5e76\u8bb0 files read<\/td>\n<td>\u4e0e\u505a\u8fc7\u2465\u7684\u7ed3\u679c\u5bf9\u6bd4&#xff0c;\u5dee\u503c\u5373&#034;\u5386\u53f2\u672a\u91cd\u6392&#034;\u7684\u4ee3\u4ef7<\/td>\n<\/tr>\n<tr>\n<td>14<\/td>\n<td>\u5efa Hive \u98ce\u683c\u5206\u6876\u8868<\/td>\n<td>5.5.2 \u4e24\u6bb5 DDL<\/td>\n<td>DESCRIBE EXTENDED \u80fd\u770b\u5230\u6876\u4fe1\u606f&#xff1b;\u5426\u5219\u8be5\u73af\u5883\u4e0d\u7528\u4e8e\u672c\u8282\u5b9e\u9a8c<\/td>\n<\/tr>\n<tr>\n<td>15<\/td>\n<td>\u5b9e\u6d4b SPJ \u662f\u5426\u751f\u6548<\/td>\n<td>5.5.2 \u9a8c\u6536 SQL&#xff0c;\u4e24\u4fa7\u6362\u6210\u6e56\u8868<\/td>\n<td>Exchange \u6d88\u5931&#061;\u751f\u6548&#xff1b;\u672a\u6d88\u5931&#061;\u6309\u65e0\u6536\u76ca\u8bbe\u8ba1&#xff0c;\u4e24\u79cd\u7ed3\u679c\u90fd\u8981\u5b58\u6863<\/td>\n<\/tr>\n<tr>\n<td>16<\/td>\n<td>\u56db\u6001\u88c1\u526a\u5b9e\u9a8c<\/td>\n<td>5.6.1 \u5efa\u8868\u704c\u6570 &#043; Q1\/Q2<\/td>\n<td>\u8868 5-10 \u81f3\u5c11\u586b\u6ee1 files read \u4e0e scanned rows \u4e24\u5217&#xff1b;\u56db\u8868 count(*) \u5b8c\u5168\u4e00\u81f4<\/td>\n<\/tr>\n<tr>\n<td>17<\/td>\n<td>\u6253\u901a\u5b9e\u6d4b\u94fe\u8def<\/td>\n<td>5.6.2 measure_scan.py<\/td>\n<td>\u80fd\u4ece\u4e8b\u4ef6\u65e5\u5fd7\u7a33\u5b9a\u6253\u51fa\u6587\u4ef6\u6570\u3001row-group \u6570\u3001scanned rows\u3001bytes read \u56db\u4e2a\u6570\u5b57<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u7b2c 6\u300112\u300113\u300115 \u9879\u662f\u672c\u7ae0\u6838\u5fc3\u3002 \u7b2c 13 \u9879\u8bc1\u660e\u7684\u662f\u672c\u7ae0\u6700\u5bb9\u6613\u88ab\u5ffd\u7565\u7684\u4e00\u6761\u56e0\u679c&#xff08;\u6392\u5e8f\u4e0d\u56de\u6eaf\u5386\u53f2&#xff09;&#xff0c;\u7b2c 15 \u9879\u51b3\u5b9a\u4f60\u540e\u9762\u6240\u6709\u6e56\u8868 join \u7684\u8bbe\u8ba1\u524d\u63d0\u2014\u2014\u8fd9\u4e24\u9879\u6ca1\u505a\u8fc7&#xff0c;\u672c\u7ae0\u7684\u5176\u4ed6\u5185\u5bb9\u90fd\u53ea\u662f\u8bfb\u8fc7\u7684\u9053\u7406\u3002<\/p>\n<hr \/>\n<h3>\u601d\u8003\u9898<\/h3>\n<li>\n<p>\u8868 5-1 \u8bf4 Iceberg \u7684\u8c13\u8bcd&#034;\u5bf9\u539f\u59cb\u5217\u5199\u5373\u53ef&#034;\u3002\u90a3\u4e48\u67e5\u8be2 WHERE report_date &gt;&#061; &#039;2026-10-01&#039; \u6253\u5728 PARTITIONED BY (days(report_date)) \u7684\u8868\u4e0a&#xff0c;\u88c1\u526a\u53d1\u751f\u5728\u7b2c\u51e0\u5c42&#xff08;\u7528\u56fe 5-2 \u7684\u56db\u5c42\u56de\u7b54&#xff09;\u3001\u7531\u8c01\u6267\u884c\u3001\u4ee5\u53ca\u4e3a\u4ec0\u4e48 WHERE speed &gt; 60 \u540c\u6837\u4e0b\u63a8\u4e86\u5374\u51e0\u4e4e\u88c1\u4e0d\u6389\u4efb\u4f55\u6587\u4ef6&#xff1f;<\/p>\n<\/li>\n<li>\n<p>\u6848\u4f8b 5 \u7684\u75c7\u72b6\u662f&#034;\u8fd4\u56de\u7a7a\u3001\u4e0d\u62a5\u9519\u3001\u800c\u4e14\u5f88\u5feb&#034;\u3002\u8bf7\u8bbe\u8ba1\u4e00\u7ec4\u4e0a\u7ebf\u524d\u65ad\u8a00&#xff0c;\u81f3\u5c11\u5305\u542b\u5206\u533a\u5b58\u5728\u6027\u3001\u7ed3\u679c\u884c\u6570\u4e0b\u9650\u3001files read \u4e0a\u9650\u4e09\u9879&#xff0c;\u5e76\u8bf4\u660e\u54ea\u4e00\u9879\u5e94\u8be5\u544a\u8b66\u3001\u54ea\u4e00\u9879\u5e94\u8be5\u76f4\u63a5\u5931\u8d25\u3001\u54ea\u4e00\u9879\u5e94\u8be5\u53ea\u5728\u7070\u5ea6\u671f\u89c2\u5bdf\u3002\u7ed9\u51fa\u7406\u7531&#xff0c;\u4e0d\u8981\u53ea\u7ed9 SQL\u3002<\/p>\n<\/li>\n<li>\n<p>\u82e5\u4e0a\u6e38\u628a VIN \u4ece&#034;11 \u4f4d\u524d\u7f00 &#043; 6 \u4f4d\u6570\u5b57&#034;\u6539\u6210&#034;20 \u4f4d\u3001\u4ec5\u672b 2 \u4f4d\u53d8\u5316&#034;\u7684\u81ea\u5b9a\u4e49\u7f16\u7801&#xff0c;\u5728 write.metadata.metrics.default &#061; truncate(16) \u4e0b&#xff0c;\u6309 vin \u7b49\u503c\u67e5\u8be2\u7684\u6587\u4ef6\u7ea7\u88c1\u526a\u4f1a\u53d1\u751f\u4ec0\u4e48&#xff1f;\u7ed9\u51fa\u4e24\u79cd\u5904\u7f6e&#xff08;\u6539 metrics \u6a21\u5f0f \/ \u6539\u6392\u5e8f\u952e&#xff09;\u5e76\u5404\u81ea\u8bf4\u660e\u4ee3\u4ef7\u843d\u5728\u54ea\u4e00\u5c42\u3002<\/p>\n<\/li>\n<li>\n<p>5.4.2 \u8bf4 Z-Order \u4f1a\u4f24\u5bb3&#034;\u6309 vin \u987a\u5e8f\u6d41\u5f0f\u5bfc\u51fa&#034;\u3002\u73b0\u5728\u4e24\u4e2a\u9700\u6c42\u90fd\u8981\u6ee1\u8db3&#xff0c;\u4e14\u4e0d\u5141\u8bb8\u62c6\u6210\u4e24\u5f20\u7269\u7406\u8868\u3002\u8bf7\u7ed9\u51fa\u4f60\u7684\u65b9\u6848&#xff08;\u53ef\u8003\u8651 Iceberg \u7684 branch\u3001\u4e0d\u540c spec\u3001\u6216\u8bfb\u5199\u5206\u79bb\u7684\u4f5c\u4e1a\u7f16\u6392&#xff09;&#xff0c;\u5e76\u8bf4\u660e\u5b83\u5f15\u5165\u4e86\u54ea\u4e9b\u65b0\u7684\u7ef4\u62a4\u6210\u672c&#xff0c;\u5f52\u5c5e\u7b2c 10 \u7ae0\u54ea\u4e00\u8282\u3002<\/p>\n<\/li>\n<li>\n<p>\u5b98\u65b9\u4e24\u53e5\u8bdd\u653e\u5728\u4e00\u8d77&#xff1a;Paimon \u7684&#034;bucket count is not a hard read-parallelism cap&#034;&#xff0c;\u4e0e\u7b2c 4 \u7ae0\u7684&#034;AQE \u53ea\u62ff shuffle \u4e4b\u540e\u7684\u7edf\u8ba1\u3001\u770b\u4e0d\u89c1\u8bfb\u4fa7&#034;\u3002\u5728&#034;20 \u4e07\u53f0\u8f66 \u00d7 \u4e00\u5929&#034;\u7684\u67e5\u8be2\u4e0a&#xff0c;\u8fd9\u4e24\u6761\u5408\u8d77\u6765\u5bf9\u4f60\u7684\u5e76\u884c\u5ea6\u8bbe\u8ba1\u610f\u5473\u7740\u4ec0\u4e48&#xff1f;\u7ed9\u51fa\u65b9\u6848\u4e0e\u9a8c\u8bc1\u6b65\u9aa4&#xff08;\u5fc5\u987b\u5305\u542b\u4e00\u4e2a\u80fd\u8bc1\u4f2a\u4f60\u81ea\u5df1\u65b9\u6848\u7684\u6307\u6807&#xff09;\u3002<\/p>\n<\/li>\n<hr \/>\n<p>\u8bfb\u5b8c\u8fd9\u4e00\u7ae0&#xff0c;\u4f60\u5e94\u8be5\u5df2\u7ecf\u80fd\u628a&#034;\u8fd9\u5f20\u8868\u6162&#034;\u7ffb\u8bd1\u6210\u4e00\u53e5\u5177\u4f53\u5f97\u591a\u7684\u8bdd&#xff1a;\u5b83\u7684 WHERE \u6761\u4ef6\u5728\u54ea\u4e00\u5c42\u843d\u7a7a\u4e86\u3002\u4e0b\u4e00\u7ae0\u4e0d\u518d\u8c08\u4f4d\u7f6e&#xff0c;\u8c08\u5185\u5bb9\u672c\u8eab\u2014\u2014\u7c7b\u578b\u9009\u9519&#xff0c;\u5b58\u50a8\u3001\u538b\u7f29\u3001\u88c1\u526a\u3001\u6b63\u786e\u6027\u56db\u5934\u4e00\u8d77\u8d54\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a\u7ae0\u8282\u5b9a\u4f4d&#xff1a;\u7b2c 4 \u7ae0\u6559\u4f60\u8bfb\u8ba1\u5212&#xff0c;\u672c\u7ae0\u6559\u4f60\u8ba9\u8ba1\u5212\u6709\u4e1c\u897f\u53ef\u88c1\u3002\u4f18\u5316\u5668\u518d\u806a\u660e&#xff0c;\u4e5f\u53ea\u80fd\u5728\u7269\u7406\u5e03\u5c40\u7ed9\u51fa\u7684\u53ef\u80fd\u6027\u91cc\u6311\u3002\u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u8fd9\u4e09\u4ef6\u4e8b\u662f\u5efa\u8868\u65f6\u4e00\u6b21\u5b9a\u6b7b\u3001\u4e8b\u540e\u6539\u8d77\u6765\u6700\u8d35\u7684\u4e09\u4e2a\u51b3\u5b9a&#xff0c;\u6240\u4ee5\u5fc5\u987b\u653e\u5728\u6e56\u4ed3\u8bfb\u5199&#xff08;\u7b2c 8-11 \u7ae0&#xff09;\u4e4b\u524d\u8bb2\u900f\u3002<br \/>\n\u672c\u7ae0\u540c\u65f6\u662f\u5168\u7cfb\u5217\u4e0e\u524d\u4f5c\u7684\u4e00\u6b21\u6b63\u5f0f\u5bf9\u9f50&#xff1a;ck \u7b2c 4 \u7ae0\u90a3\u5f20\\&#8221;\u5206\u533a \u2192 \u4e3b\u952e \u2192 \u8df3\u6570\u7d22\u5f15 \u2192 \u9884\u8ba1\u7b97\\&#8221;\u7684<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[365,323],"topic":[],"class_list":["post-124025","post","type-post","status-publish","format-standard","hentry","category-server","tag-spark","tag-323"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/124025.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a\u7ae0\u8282\u5b9a\u4f4d&#xff1a;\u7b2c 4 \u7ae0\u6559\u4f60\u8bfb\u8ba1\u5212&#xff0c;\u672c\u7ae0\u6559\u4f60\u8ba9\u8ba1\u5212\u6709\u4e1c\u897f\u53ef\u88c1\u3002\u4f18\u5316\u5668\u518d\u806a\u660e&#xff0c;\u4e5f\u53ea\u80fd\u5728\u7269\u7406\u5e03\u5c40\u7ed9\u51fa\u7684\u53ef\u80fd\u6027\u91cc\u6311\u3002\u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u8fd9\u4e09\u4ef6\u4e8b\u662f\u5efa\u8868\u65f6\u4e00\u6b21\u5b9a\u6b7b\u3001\u4e8b\u540e\u6539\u8d77\u6765\u6700\u8d35\u7684\u4e09\u4e2a\u51b3\u5b9a&#xff0c;\u6240\u4ee5\u5fc5\u987b\u653e\u5728\u6e56\u4ed3\u8bfb\u5199&#xff08;\u7b2c 8-11 \u7ae0&#xff09;\u4e4b\u524d\u8bb2\u900f\u3002 \u672c\u7ae0\u540c\u65f6\u662f\u5168\u7cfb\u5217\u4e0e\u524d\u4f5c\u7684\u4e00\u6b21\u6b63\u5f0f\u5bf9\u9f50&#xff1a;ck \u7b2c 4 \u7ae0\u90a3\u5f20&quot;\u5206\u533a \u2192 \u4e3b\u952e \u2192 \u8df3\u6570\u7d22\u5f15 \u2192 \u9884\u8ba1\u7b97&quot;\u7684\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/124025.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-11-08T11:40:55+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"38 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/124025.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/124025.html\",\"name\":\"\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-11-08T11:40:55+00:00\",\"dateModified\":\"2026-11-08T11:40:55+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/124025.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/124025.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/124025.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/124025.html","og_locale":"zh_CN","og_type":"article","og_title":"\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a\u7ae0\u8282\u5b9a\u4f4d&#xff1a;\u7b2c 4 \u7ae0\u6559\u4f60\u8bfb\u8ba1\u5212&#xff0c;\u672c\u7ae0\u6559\u4f60\u8ba9\u8ba1\u5212\u6709\u4e1c\u897f\u53ef\u88c1\u3002\u4f18\u5316\u5668\u518d\u806a\u660e&#xff0c;\u4e5f\u53ea\u80fd\u5728\u7269\u7406\u5e03\u5c40\u7ed9\u51fa\u7684\u53ef\u80fd\u6027\u91cc\u6311\u3002\u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u8fd9\u4e09\u4ef6\u4e8b\u662f\u5efa\u8868\u65f6\u4e00\u6b21\u5b9a\u6b7b\u3001\u4e8b\u540e\u6539\u8d77\u6765\u6700\u8d35\u7684\u4e09\u4e2a\u51b3\u5b9a&#xff0c;\u6240\u4ee5\u5fc5\u987b\u653e\u5728\u6e56\u4ed3\u8bfb\u5199&#xff08;\u7b2c 8-11 \u7ae0&#xff09;\u4e4b\u524d\u8bb2\u900f\u3002 \u672c\u7ae0\u540c\u65f6\u662f\u5168\u7cfb\u5217\u4e0e\u524d\u4f5c\u7684\u4e00\u6b21\u6b63\u5f0f\u5bf9\u9f50&#xff1a;ck \u7b2c 4 \u7ae0\u90a3\u5f20\"\u5206\u533a \u2192 \u4e3b\u952e \u2192 \u8df3\u6570\u7d22\u5f15 \u2192 \u9884\u8ba1\u7b97\"\u7684","og_url":"https:\/\/www.wsisp.com\/helps\/124025.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-11-08T11:40:55+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"38 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/124025.html","url":"https:\/\/www.wsisp.com\/helps\/124025.html","name":"\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-11-08T11:40:55+00:00","dateModified":"2026-11-08T11:40:55+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/124025.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/124025.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/124025.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u7b2c5\u7ae0 Spark4.x \u5206\u533a\u3001\u5206\u6876\u3001\u6392\u5e8f\u4e0e\u6570\u636e\u88c1\u526a"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/124025","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=124025"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/124025\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=124025"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=124025"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=124025"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=124025"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}